[Fix] inkling: Return 400 instead of 500 for unfetchable/undecodable multimodal input (#40899)

Co-authored-by: apex-mochen <265981728+apex-mochen@users.noreply.github.com>
Co-authored-by: Po-Han Huang (NVIDIA) <53919306+nvpohanh@users.noreply.github.com>
Co-authored-by: apex-mochen <2756823972@qq.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
Co-authored-by: Po-Han Huang <pohanh@nvidia.com>
This commit is contained in:
hanahhh
2026-10-20 09:18:18 +08:00
committed by GitHub
co-authored by apex-mochen Po-Han Huang apex-mochen Claude Opus 5.5 Po-Han Huang
parent 693d333306
commit 71ee2dc950
5 changed files with 369 additions and 15 deletions
@@ -2,6 +2,7 @@
from __future__ import annotations
import errno
import io
import math
from dataclasses import dataclass
@@ -29,6 +30,10 @@ class InklingAudioEncoderParams:
audio_token_duration_s: float = 0.05
# Anything else (EMFILE, ENOSPC, EACCES, ...) is a server-side fault, not a bad path.
_EXPECTED_PATH_ERRNOS = {errno.ENOENT, errno.ENOTDIR, errno.EISDIR, errno.ENAMETOOLONG}
def _load_audio_bytes(audio) -> bytes:
"""Coerce a single audio input into raw file bytes for the encoder.
@@ -41,8 +46,19 @@ def _load_audio_bytes(audio) -> bytes:
return audio.read()
if isinstance(audio, str):
path = audio[len("file://") :] if audio.startswith("file://") else audio
with open(path, "rb") as f:
return f.read()
try:
with open(path, "rb") as f:
return f.read()
except OSError as e:
if e.errno not in _EXPECTED_PATH_ERRNOS:
raise
path_str = path
if len(path_str) > 100:
path_str = path_str[:100] + "..."
# str(e) re-embeds the full, untruncated path; use e.strerror instead.
raise ValueError(
f"Could not read audio from path {path_str}: {e.strerror}"
) from e
raise TypeError(
f"Unsupported audio input type for Inkling audio extractor: {type(audio)}"
)
@@ -56,9 +72,12 @@ def _to_exact_int(value: float, name: str, tolerance: float = 1e-6) -> int:
def _decode_audio(audio_bytes: bytes, sample_rate: int) -> torch.Tensor:
samples, src_sample_rate = sf.read(
io.BytesIO(audio_bytes), dtype="float32", always_2d=True
)
try:
samples, src_sample_rate = sf.read(
io.BytesIO(audio_bytes), dtype="float32", always_2d=True
)
except sf.LibsndfileError as e:
raise ValueError(f"Could not decode audio: {e}") from e
mono = samples.mean(axis=1)
if src_sample_rate != sample_rate:
mono = _resample(mono, src_sample_rate, sample_rate)
@@ -2,6 +2,7 @@
from __future__ import annotations
import errno
import io
import math
from typing import List, Optional, Union
@@ -17,6 +18,9 @@ IMAGE_STD = np.array([0.26862954, 0.2613026, 0.2757771], dtype=np.float32)
PAD_RAW_VALUE = np.float32(-1.0 / 255.0)
PAD_NORM = (np.full((3,), PAD_RAW_VALUE, dtype=np.float32) - IMAGE_MEAN) / IMAGE_STD
# Anything else (EMFILE, ENOSPC, EACCES, ...) is a server-side fault, not a bad path.
_EXPECTED_PATH_ERRNOS = {errno.ENOENT, errno.ENOTDIR, errno.EISDIR, errno.ENAMETOOLONG}
def _validate_image_rescale(
rescale_image_frac: Optional[float],
@@ -86,8 +90,19 @@ def _load_image_bytes(image) -> bytes:
"upstream (e.g. via SGLang load_mm_data) before preprocessing."
)
path = image[len("file://") :] if image.startswith("file://") else image
with open(path, "rb") as f:
return f.read()
try:
with open(path, "rb") as f:
return f.read()
except OSError as e:
if e.errno not in _EXPECTED_PATH_ERRNOS:
raise
path_str = path
if len(path_str) > 100:
path_str = path_str[:100] + "..."
# str(e) re-embeds the full, untruncated path; use e.strerror instead.
raise ValueError(
f"Could not read image from path {path_str}: {e.strerror}"
) from e
from PIL import Image
@@ -153,7 +168,10 @@ def _encode_image_bytes(
from PIL import Image
image = Image.open(io.BytesIO(image_bytes)).convert("RGB")
try:
image = Image.open(io.BytesIO(image_bytes)).convert("RGB")
except (OSError, SyntaxError) as e:
raise ValueError(f"Could not decode image: {e}") from e
scaled_size = _scaled_image_dimensions(
*image.size,
rescale_image_frac=rescale_image_frac,
@@ -34,7 +34,11 @@ _pil_pool = ThreadPoolExecutor(max_workers=8)
def _pil_decode(raw: bytes) -> np.ndarray:
return np.ascontiguousarray(np.array(Image.open(io.BytesIO(raw)).convert("RGB")))
try:
image = Image.open(io.BytesIO(raw)).convert("RGB")
except (OSError, SyntaxError) as e:
raise ValueError(f"Could not decode image: {e}") from e
return np.ascontiguousarray(np.array(image))
class InklingRustImageProcessor(BaseImageProcessor):
@@ -52,7 +52,7 @@ from sglang.srt.parser.inkling_tokenizer import IMAGE_TOKEN_ID as INKLING_IMAGE_
from sglang.srt.parser.inkling_tokenizer import (
INKLING_SPECIAL_TOKEN_IDS,
)
from sglang.srt.utils.common import download_remote_media
from sglang.srt.utils.common import CLIENT_MEDIA_EXCEPTIONS, download_remote_media
logger = logging.getLogger(__name__)
@@ -78,11 +78,21 @@ def _resolve_media_item(item):
url = getattr(item, "url")
if not isinstance(url, str):
return item
if url.startswith("data:"):
header, _, payload = url.partition(",")
return base64.b64decode(payload) if ";base64" in header else payload.encode()
if url.startswith(("http://", "https://")):
return download_remote_media(url, timeout=30)
if not url.strip():
raise ValueError("empty media URL")
try:
if url.startswith("data:"):
header, _, payload = url.partition(",")
return (
base64.b64decode(payload) if ";base64" in header else payload.encode()
)
if url.startswith(("http://", "https://")):
return download_remote_media(url, timeout=30)
except CLIENT_MEDIA_EXCEPTIONS as e:
url_str = url
if len(url_str) > 100:
url_str = url_str[:100] + "..."
raise ValueError(f"Error while resolving media {url_str}: {e}") from e
return url # plain path / file:// -> handled by the per-modality byte loader
@@ -0,0 +1,303 @@
"""Unit tests for client-error classification in Inkling's multimodal fetch/decode path."""
from sglang.test.ci.ci_register import register_cpu_ci
register_cpu_ci(est_time=8, suite="base-a-test-cpu")
import errno
import io
import tempfile
import unittest
from pathlib import Path
from unittest.mock import MagicMock, patch
import numpy as np
import requests
import soundfile as sf
from PIL import Image
from sglang.srt.multimodal.inkling.feature_extraction import (
_decode_audio,
_load_audio_bytes,
)
from sglang.srt.multimodal.inkling.image_processing import (
_encode_image_bytes,
_load_image_bytes,
)
from sglang.srt.multimodal.inkling.image_processing_rust import _pil_decode
from sglang.srt.multimodal.processors.inkling import _resolve_media_item
from sglang.test.test_utils import CustomTestCase
def _session_raising(exc):
session = MagicMock()
session.get.side_effect = exc
return session
def _png_bytes(width: int = 16, height: int = 16) -> bytes:
arr = (np.random.RandomState(0).rand(height, width, 3) * 255).astype("uint8")
buf = io.BytesIO()
Image.fromarray(arr, "RGB").save(buf, format="PNG")
return buf.getvalue()
def _broken_chunk_png_bytes() -> bytes:
"""A structurally-valid PNG with a corrupted IDAT chunk length byte.
PIL raises SyntaxError (not OSError) for this, matching the fixture in
test_common.py::TestLoadImage.
"""
broken = bytearray(_png_bytes())
idat_pos = broken.index(b"IDAT")
broken[idat_pos - 1] -= 6
return bytes(broken)
class TestResolveMediaItemIsClientError(CustomTestCase):
"""_resolve_media_item() -- empty/unreachable media URL (image or audio)."""
def test_empty_url_string_raises_value_error(self):
with self.assertRaisesRegex(ValueError, "empty media URL"):
_resolve_media_item("")
def test_empty_url_in_mapping_raises_value_error(self):
with self.assertRaisesRegex(ValueError, "empty media URL"):
_resolve_media_item({"url": ""})
def test_empty_url_attribute_raises_value_error(self):
class _ImageData:
url = ""
with self.assertRaisesRegex(ValueError, "empty media URL"):
_resolve_media_item(_ImageData())
def test_whitespace_only_url_raises_value_error(self):
# A client sending " " instead of "" must be rejected the same way,
# not silently fall through to being treated as a plain file path.
for whitespace_url in (" ", "\t", "\n", " \t\n "):
with self.subTest(url=repr(whitespace_url)):
with self.assertRaisesRegex(ValueError, "empty media URL"):
_resolve_media_item({"url": whitespace_url})
def test_unfetchable_url_every_request_exception(self):
# download_remote_media() fetches through get_mm_http_session(); HTTPError,
# ConnectionError and Timeout all subclass RequestException.
for exc in (
requests.exceptions.HTTPError("404 from media host"),
requests.exceptions.ConnectionError("dns failure"),
requests.exceptions.Timeout("read timed out"),
):
with self.subTest(exc=type(exc).__name__):
with patch(
"sglang.srt.utils.common.get_mm_http_session",
return_value=_session_raising(exc),
):
with self.assertRaises(ValueError) as ctx:
_resolve_media_item({"url": "https://media.host/clip.png"})
# the raw RequestException must not leak past the boundary
self.assertNotIsInstance(
ctx.exception, requests.exceptions.RequestException
)
self.assertIsInstance(ctx.exception.__cause__, type(exc))
def test_long_url_error_message_is_truncated(self):
# A "data:" URL's base64 payload (or a pathologically long http(s) URL)
# can be megabytes; the error message (logged, and echoed in the 400
# response) must stay bounded. Route through http(s) + a mocked fetch
# failure so this doesn't depend on the payload happening to be valid
# base64 (a long run of "A"s decodes cleanly and wouldn't raise).
huge_url = "https://media.host/" + ("a" * 1_000_000)
with patch(
"sglang.srt.utils.common.get_mm_http_session",
return_value=_session_raising(requests.exceptions.ConnectionError("x")),
):
with self.assertRaises(ValueError) as ctx:
_resolve_media_item({"url": huge_url})
self.assertLess(len(str(ctx.exception)), 200)
def test_non_url_item_passes_through_unchanged(self):
# Non-media, non-string, non-Mapping items (already-resolved bytes, PIL
# images, etc.) must not be touched by the empty-url / fetch guards.
sentinel = b"already-resolved-bytes"
self.assertIs(_resolve_media_item(sentinel), sentinel)
def test_plain_path_still_passes_through(self):
# A local path / file:// URI is intentionally not resolved here -- it's
# handled by the per-modality byte loader downstream -- so it must not
# be mistaken for an "empty" or "unfetchable" input.
self.assertEqual(
_resolve_media_item("/tmp/some-local-image.png"),
"/tmp/some-local-image.png",
)
class TestEncodeImageBytesIsClientError(CustomTestCase):
"""_encode_image_bytes() (image_processing.py) -- plain-Python image decode path."""
def _encode(self, image_bytes: bytes):
return _encode_image_bytes(
image_bytes,
patch_size=14,
rescale_image_frac=None,
rescale_image_max_upscaled_long_edge=None,
)
def test_undecodable_image_bytes_raises_value_error(self):
# PIL raises UnidentifiedImageError, an OSError -- not a ValueError --
# for bytes it cannot identify as any supported image format.
with self.assertRaises(ValueError) as ctx:
self._encode(b"definitely not an image")
self.assertIn("Could not decode image", str(ctx.exception))
self.assertIsInstance(ctx.exception.__cause__, OSError)
def test_truncated_image_raises_value_error(self):
# A recognizable-but-corrupt header (truncated PNG signature) must also
# be reclassified, not just a fully unrecognized format.
with self.assertRaisesRegex(ValueError, "Could not decode image"):
self._encode(b"\x89PNG\r\n\x1a\n")
def test_broken_chunk_png_raises_value_error(self):
# PIL raises SyntaxError (not OSError) for a structurally-valid PNG
# with a corrupted chunk; must be reclassified too.
with self.assertRaises(ValueError) as ctx:
self._encode(_broken_chunk_png_bytes())
self.assertIsInstance(ctx.exception.__cause__, SyntaxError)
def test_valid_image_still_decodes(self):
# Regression guard: the try/except must not change behavior for valid input.
result = self._encode(_png_bytes())
self.assertEqual(result.shape[-1], 3)
class TestPilDecodeIsClientError(CustomTestCase):
"""_pil_decode() (image_processing_rust.py) -- the default image decode path."""
def test_undecodable_image_bytes_raises_value_error(self):
with self.assertRaises(ValueError) as ctx:
_pil_decode(b"definitely not an image")
self.assertIn("Could not decode image", str(ctx.exception))
self.assertIsInstance(ctx.exception.__cause__, OSError)
def test_broken_chunk_png_raises_value_error(self):
with self.assertRaises(ValueError) as ctx:
_pil_decode(_broken_chunk_png_bytes())
self.assertIsInstance(ctx.exception.__cause__, SyntaxError)
def test_valid_image_still_decodes(self):
result = _pil_decode(_png_bytes())
self.assertEqual(result.shape[-1], 3)
class TestLoadImageBytesIsClientError(CustomTestCase):
"""_load_image_bytes() (image_processing.py) -- shared by both image decode paths."""
def test_missing_local_path_raises_value_error(self):
with self.assertRaises(ValueError) as ctx:
_load_image_bytes("/nonexistent/path/that/does/not/exist.png")
self.assertIn("Could not read image from path", str(ctx.exception))
self.assertIsInstance(ctx.exception.__cause__, OSError)
def test_path_component_that_is_not_a_directory_raises_value_error(self):
with tempfile.TemporaryDirectory() as temp_dir:
not_a_directory = Path(temp_dir) / "file"
not_a_directory.write_bytes(b"x")
with self.assertRaisesRegex(ValueError, "Could not read image from path"):
_load_image_bytes(str(not_a_directory / "child.png"))
def test_directory_path_raises_value_error(self):
with (
tempfile.TemporaryDirectory() as temp_dir,
self.assertRaisesRegex(ValueError, "Could not read image from path"),
):
_load_image_bytes(temp_dir)
def test_overlong_path_raises_value_error(self):
# A raw base64 payload sent without the "data:" prefix falls through
# _resolve_media_item() as a "plain path" and ends up here.
with self.assertRaises(ValueError) as ctx:
_load_image_bytes("a" * 300)
self.assertIn("Could not read image from path", str(ctx.exception))
self.assertLess(len(str(ctx.exception)), 200)
def test_server_os_error_is_not_reclassified_as_client_error(self):
with (
patch(
"builtins.open",
side_effect=OSError(errno.EMFILE, "too many open files"),
),
self.assertRaises(OSError) as ctx,
):
_load_image_bytes("/any/image.png")
self.assertEqual(ctx.exception.errno, errno.EMFILE)
class TestLoadAudioBytesIsClientError(CustomTestCase):
"""_load_audio_bytes() (feature_extraction.py) -- same gap as the image loader
above, for the audio modality."""
def test_missing_local_path_raises_value_error(self):
with self.assertRaises(ValueError) as ctx:
_load_audio_bytes("/nonexistent/path/that/does/not/exist.wav")
self.assertIn("Could not read audio from path", str(ctx.exception))
self.assertIsInstance(ctx.exception.__cause__, OSError)
def test_path_component_that_is_not_a_directory_raises_value_error(self):
with tempfile.TemporaryDirectory() as temp_dir:
not_a_directory = Path(temp_dir) / "file"
not_a_directory.write_bytes(b"x")
with self.assertRaisesRegex(ValueError, "Could not read audio from path"):
_load_audio_bytes(str(not_a_directory / "child.wav"))
def test_directory_path_raises_value_error(self):
with (
tempfile.TemporaryDirectory() as temp_dir,
self.assertRaisesRegex(ValueError, "Could not read audio from path"),
):
_load_audio_bytes(temp_dir)
def test_overlong_path_raises_value_error(self):
with self.assertRaises(ValueError) as ctx:
_load_audio_bytes("a" * 300)
self.assertIn("Could not read audio from path", str(ctx.exception))
self.assertLess(len(str(ctx.exception)), 200)
def test_server_os_error_is_not_reclassified_as_client_error(self):
with (
patch(
"builtins.open",
side_effect=OSError(errno.EMFILE, "too many open files"),
),
self.assertRaises(OSError) as ctx,
):
_load_audio_bytes("/any/audio.wav")
self.assertEqual(ctx.exception.errno, errno.EMFILE)
class TestDecodeAudioIsClientError(CustomTestCase):
"""_decode_audio() (feature_extraction.py) -- undecodable audio bytes.
soundfile raises LibsndfileError, a RuntimeError subclass, so it must be
explicitly caught and reclassified."""
def test_undecodable_audio_bytes_raises_value_error(self):
with self.assertRaises(ValueError) as ctx:
_decode_audio(b"definitely not audio", sample_rate=16000)
self.assertIn("Could not decode audio", str(ctx.exception))
self.assertIsInstance(ctx.exception.__cause__, sf.LibsndfileError)
def test_valid_audio_still_decodes(self):
# Regression guard: a 1-second 440Hz tone should still decode without
# raising, and resampling (source rate == target rate here, so it's a
# no-op) must not be affected either.
sample_rate = 16000
t = np.linspace(0, 1, sample_rate, endpoint=False)
tone = (0.1 * np.sin(2 * np.pi * 440 * t)).astype(np.float32)
buf = io.BytesIO()
sf.write(buf, tone, sample_rate, format="WAV")
result = _decode_audio(buf.getvalue(), sample_rate=sample_rate)
self.assertEqual(result.shape[0], sample_rate)
if __name__ == "__main__":
unittest.main()