diff --git a/python/sglang/srt/multimodal/inkling/feature_extraction.py b/python/sglang/srt/multimodal/inkling/feature_extraction.py index faaa1ff1e3..eba04f5e0f 100644 --- a/python/sglang/srt/multimodal/inkling/feature_extraction.py +++ b/python/sglang/srt/multimodal/inkling/feature_extraction.py @@ -2,6 +2,7 @@ from __future__ import annotations +import errno import io import math from dataclasses import dataclass @@ -29,6 +30,10 @@ class InklingAudioEncoderParams: audio_token_duration_s: float = 0.05 +# Anything else (EMFILE, ENOSPC, EACCES, ...) is a server-side fault, not a bad path. +_EXPECTED_PATH_ERRNOS = {errno.ENOENT, errno.ENOTDIR, errno.EISDIR, errno.ENAMETOOLONG} + + def _load_audio_bytes(audio) -> bytes: """Coerce a single audio input into raw file bytes for the encoder. @@ -41,8 +46,19 @@ def _load_audio_bytes(audio) -> bytes: return audio.read() if isinstance(audio, str): path = audio[len("file://") :] if audio.startswith("file://") else audio - with open(path, "rb") as f: - return f.read() + try: + with open(path, "rb") as f: + return f.read() + except OSError as e: + if e.errno not in _EXPECTED_PATH_ERRNOS: + raise + path_str = path + if len(path_str) > 100: + path_str = path_str[:100] + "..." + # str(e) re-embeds the full, untruncated path; use e.strerror instead. + raise ValueError( + f"Could not read audio from path {path_str}: {e.strerror}" + ) from e raise TypeError( f"Unsupported audio input type for Inkling audio extractor: {type(audio)}" ) @@ -56,9 +72,12 @@ def _to_exact_int(value: float, name: str, tolerance: float = 1e-6) -> int: def _decode_audio(audio_bytes: bytes, sample_rate: int) -> torch.Tensor: - samples, src_sample_rate = sf.read( - io.BytesIO(audio_bytes), dtype="float32", always_2d=True - ) + try: + samples, src_sample_rate = sf.read( + io.BytesIO(audio_bytes), dtype="float32", always_2d=True + ) + except sf.LibsndfileError as e: + raise ValueError(f"Could not decode audio: {e}") from e mono = samples.mean(axis=1) if src_sample_rate != sample_rate: mono = _resample(mono, src_sample_rate, sample_rate) diff --git a/python/sglang/srt/multimodal/inkling/image_processing.py b/python/sglang/srt/multimodal/inkling/image_processing.py index 1a99d45e3e..18447b07eb 100644 --- a/python/sglang/srt/multimodal/inkling/image_processing.py +++ b/python/sglang/srt/multimodal/inkling/image_processing.py @@ -2,6 +2,7 @@ from __future__ import annotations +import errno import io import math from typing import List, Optional, Union @@ -17,6 +18,9 @@ IMAGE_STD = np.array([0.26862954, 0.2613026, 0.2757771], dtype=np.float32) PAD_RAW_VALUE = np.float32(-1.0 / 255.0) PAD_NORM = (np.full((3,), PAD_RAW_VALUE, dtype=np.float32) - IMAGE_MEAN) / IMAGE_STD +# Anything else (EMFILE, ENOSPC, EACCES, ...) is a server-side fault, not a bad path. +_EXPECTED_PATH_ERRNOS = {errno.ENOENT, errno.ENOTDIR, errno.EISDIR, errno.ENAMETOOLONG} + def _validate_image_rescale( rescale_image_frac: Optional[float], @@ -86,8 +90,19 @@ def _load_image_bytes(image) -> bytes: "upstream (e.g. via SGLang load_mm_data) before preprocessing." ) path = image[len("file://") :] if image.startswith("file://") else image - with open(path, "rb") as f: - return f.read() + try: + with open(path, "rb") as f: + return f.read() + except OSError as e: + if e.errno not in _EXPECTED_PATH_ERRNOS: + raise + path_str = path + if len(path_str) > 100: + path_str = path_str[:100] + "..." + # str(e) re-embeds the full, untruncated path; use e.strerror instead. + raise ValueError( + f"Could not read image from path {path_str}: {e.strerror}" + ) from e from PIL import Image @@ -153,7 +168,10 @@ def _encode_image_bytes( from PIL import Image - image = Image.open(io.BytesIO(image_bytes)).convert("RGB") + try: + image = Image.open(io.BytesIO(image_bytes)).convert("RGB") + except (OSError, SyntaxError) as e: + raise ValueError(f"Could not decode image: {e}") from e scaled_size = _scaled_image_dimensions( *image.size, rescale_image_frac=rescale_image_frac, diff --git a/python/sglang/srt/multimodal/inkling/image_processing_rust.py b/python/sglang/srt/multimodal/inkling/image_processing_rust.py index 55a439c48c..13f0c65a0c 100644 --- a/python/sglang/srt/multimodal/inkling/image_processing_rust.py +++ b/python/sglang/srt/multimodal/inkling/image_processing_rust.py @@ -34,7 +34,11 @@ _pil_pool = ThreadPoolExecutor(max_workers=8) def _pil_decode(raw: bytes) -> np.ndarray: - return np.ascontiguousarray(np.array(Image.open(io.BytesIO(raw)).convert("RGB"))) + try: + image = Image.open(io.BytesIO(raw)).convert("RGB") + except (OSError, SyntaxError) as e: + raise ValueError(f"Could not decode image: {e}") from e + return np.ascontiguousarray(np.array(image)) class InklingRustImageProcessor(BaseImageProcessor): diff --git a/python/sglang/srt/multimodal/processors/inkling.py b/python/sglang/srt/multimodal/processors/inkling.py index 803e973228..7aaa8cf8f2 100644 --- a/python/sglang/srt/multimodal/processors/inkling.py +++ b/python/sglang/srt/multimodal/processors/inkling.py @@ -52,7 +52,7 @@ from sglang.srt.parser.inkling_tokenizer import IMAGE_TOKEN_ID as INKLING_IMAGE_ from sglang.srt.parser.inkling_tokenizer import ( INKLING_SPECIAL_TOKEN_IDS, ) -from sglang.srt.utils.common import download_remote_media +from sglang.srt.utils.common import CLIENT_MEDIA_EXCEPTIONS, download_remote_media logger = logging.getLogger(__name__) @@ -78,11 +78,21 @@ def _resolve_media_item(item): url = getattr(item, "url") if not isinstance(url, str): return item - if url.startswith("data:"): - header, _, payload = url.partition(",") - return base64.b64decode(payload) if ";base64" in header else payload.encode() - if url.startswith(("http://", "https://")): - return download_remote_media(url, timeout=30) + if not url.strip(): + raise ValueError("empty media URL") + try: + if url.startswith("data:"): + header, _, payload = url.partition(",") + return ( + base64.b64decode(payload) if ";base64" in header else payload.encode() + ) + if url.startswith(("http://", "https://")): + return download_remote_media(url, timeout=30) + except CLIENT_MEDIA_EXCEPTIONS as e: + url_str = url + if len(url_str) > 100: + url_str = url_str[:100] + "..." + raise ValueError(f"Error while resolving media {url_str}: {e}") from e return url # plain path / file:// -> handled by the per-modality byte loader diff --git a/test/registered/unit/multimodal/test_inkling_media_errors.py b/test/registered/unit/multimodal/test_inkling_media_errors.py new file mode 100644 index 0000000000..80f6e2ede3 --- /dev/null +++ b/test/registered/unit/multimodal/test_inkling_media_errors.py @@ -0,0 +1,303 @@ +"""Unit tests for client-error classification in Inkling's multimodal fetch/decode path.""" + +from sglang.test.ci.ci_register import register_cpu_ci + +register_cpu_ci(est_time=8, suite="base-a-test-cpu") + +import errno +import io +import tempfile +import unittest +from pathlib import Path +from unittest.mock import MagicMock, patch + +import numpy as np +import requests +import soundfile as sf +from PIL import Image + +from sglang.srt.multimodal.inkling.feature_extraction import ( + _decode_audio, + _load_audio_bytes, +) +from sglang.srt.multimodal.inkling.image_processing import ( + _encode_image_bytes, + _load_image_bytes, +) +from sglang.srt.multimodal.inkling.image_processing_rust import _pil_decode +from sglang.srt.multimodal.processors.inkling import _resolve_media_item +from sglang.test.test_utils import CustomTestCase + + +def _session_raising(exc): + session = MagicMock() + session.get.side_effect = exc + return session + + +def _png_bytes(width: int = 16, height: int = 16) -> bytes: + arr = (np.random.RandomState(0).rand(height, width, 3) * 255).astype("uint8") + buf = io.BytesIO() + Image.fromarray(arr, "RGB").save(buf, format="PNG") + return buf.getvalue() + + +def _broken_chunk_png_bytes() -> bytes: + """A structurally-valid PNG with a corrupted IDAT chunk length byte. + + PIL raises SyntaxError (not OSError) for this, matching the fixture in + test_common.py::TestLoadImage. + """ + broken = bytearray(_png_bytes()) + idat_pos = broken.index(b"IDAT") + broken[idat_pos - 1] -= 6 + return bytes(broken) + + +class TestResolveMediaItemIsClientError(CustomTestCase): + """_resolve_media_item() -- empty/unreachable media URL (image or audio).""" + + def test_empty_url_string_raises_value_error(self): + with self.assertRaisesRegex(ValueError, "empty media URL"): + _resolve_media_item("") + + def test_empty_url_in_mapping_raises_value_error(self): + with self.assertRaisesRegex(ValueError, "empty media URL"): + _resolve_media_item({"url": ""}) + + def test_empty_url_attribute_raises_value_error(self): + class _ImageData: + url = "" + + with self.assertRaisesRegex(ValueError, "empty media URL"): + _resolve_media_item(_ImageData()) + + def test_whitespace_only_url_raises_value_error(self): + # A client sending " " instead of "" must be rejected the same way, + # not silently fall through to being treated as a plain file path. + for whitespace_url in (" ", "\t", "\n", " \t\n "): + with self.subTest(url=repr(whitespace_url)): + with self.assertRaisesRegex(ValueError, "empty media URL"): + _resolve_media_item({"url": whitespace_url}) + + def test_unfetchable_url_every_request_exception(self): + # download_remote_media() fetches through get_mm_http_session(); HTTPError, + # ConnectionError and Timeout all subclass RequestException. + for exc in ( + requests.exceptions.HTTPError("404 from media host"), + requests.exceptions.ConnectionError("dns failure"), + requests.exceptions.Timeout("read timed out"), + ): + with self.subTest(exc=type(exc).__name__): + with patch( + "sglang.srt.utils.common.get_mm_http_session", + return_value=_session_raising(exc), + ): + with self.assertRaises(ValueError) as ctx: + _resolve_media_item({"url": "https://media.host/clip.png"}) + # the raw RequestException must not leak past the boundary + self.assertNotIsInstance( + ctx.exception, requests.exceptions.RequestException + ) + self.assertIsInstance(ctx.exception.__cause__, type(exc)) + + def test_long_url_error_message_is_truncated(self): + # A "data:" URL's base64 payload (or a pathologically long http(s) URL) + # can be megabytes; the error message (logged, and echoed in the 400 + # response) must stay bounded. Route through http(s) + a mocked fetch + # failure so this doesn't depend on the payload happening to be valid + # base64 (a long run of "A"s decodes cleanly and wouldn't raise). + huge_url = "https://media.host/" + ("a" * 1_000_000) + with patch( + "sglang.srt.utils.common.get_mm_http_session", + return_value=_session_raising(requests.exceptions.ConnectionError("x")), + ): + with self.assertRaises(ValueError) as ctx: + _resolve_media_item({"url": huge_url}) + self.assertLess(len(str(ctx.exception)), 200) + + def test_non_url_item_passes_through_unchanged(self): + # Non-media, non-string, non-Mapping items (already-resolved bytes, PIL + # images, etc.) must not be touched by the empty-url / fetch guards. + sentinel = b"already-resolved-bytes" + self.assertIs(_resolve_media_item(sentinel), sentinel) + + def test_plain_path_still_passes_through(self): + # A local path / file:// URI is intentionally not resolved here -- it's + # handled by the per-modality byte loader downstream -- so it must not + # be mistaken for an "empty" or "unfetchable" input. + self.assertEqual( + _resolve_media_item("/tmp/some-local-image.png"), + "/tmp/some-local-image.png", + ) + + +class TestEncodeImageBytesIsClientError(CustomTestCase): + """_encode_image_bytes() (image_processing.py) -- plain-Python image decode path.""" + + def _encode(self, image_bytes: bytes): + return _encode_image_bytes( + image_bytes, + patch_size=14, + rescale_image_frac=None, + rescale_image_max_upscaled_long_edge=None, + ) + + def test_undecodable_image_bytes_raises_value_error(self): + # PIL raises UnidentifiedImageError, an OSError -- not a ValueError -- + # for bytes it cannot identify as any supported image format. + with self.assertRaises(ValueError) as ctx: + self._encode(b"definitely not an image") + self.assertIn("Could not decode image", str(ctx.exception)) + self.assertIsInstance(ctx.exception.__cause__, OSError) + + def test_truncated_image_raises_value_error(self): + # A recognizable-but-corrupt header (truncated PNG signature) must also + # be reclassified, not just a fully unrecognized format. + with self.assertRaisesRegex(ValueError, "Could not decode image"): + self._encode(b"\x89PNG\r\n\x1a\n") + + def test_broken_chunk_png_raises_value_error(self): + # PIL raises SyntaxError (not OSError) for a structurally-valid PNG + # with a corrupted chunk; must be reclassified too. + with self.assertRaises(ValueError) as ctx: + self._encode(_broken_chunk_png_bytes()) + self.assertIsInstance(ctx.exception.__cause__, SyntaxError) + + def test_valid_image_still_decodes(self): + # Regression guard: the try/except must not change behavior for valid input. + result = self._encode(_png_bytes()) + self.assertEqual(result.shape[-1], 3) + + +class TestPilDecodeIsClientError(CustomTestCase): + """_pil_decode() (image_processing_rust.py) -- the default image decode path.""" + + def test_undecodable_image_bytes_raises_value_error(self): + with self.assertRaises(ValueError) as ctx: + _pil_decode(b"definitely not an image") + self.assertIn("Could not decode image", str(ctx.exception)) + self.assertIsInstance(ctx.exception.__cause__, OSError) + + def test_broken_chunk_png_raises_value_error(self): + with self.assertRaises(ValueError) as ctx: + _pil_decode(_broken_chunk_png_bytes()) + self.assertIsInstance(ctx.exception.__cause__, SyntaxError) + + def test_valid_image_still_decodes(self): + result = _pil_decode(_png_bytes()) + self.assertEqual(result.shape[-1], 3) + + +class TestLoadImageBytesIsClientError(CustomTestCase): + """_load_image_bytes() (image_processing.py) -- shared by both image decode paths.""" + + def test_missing_local_path_raises_value_error(self): + with self.assertRaises(ValueError) as ctx: + _load_image_bytes("/nonexistent/path/that/does/not/exist.png") + self.assertIn("Could not read image from path", str(ctx.exception)) + self.assertIsInstance(ctx.exception.__cause__, OSError) + + def test_path_component_that_is_not_a_directory_raises_value_error(self): + with tempfile.TemporaryDirectory() as temp_dir: + not_a_directory = Path(temp_dir) / "file" + not_a_directory.write_bytes(b"x") + with self.assertRaisesRegex(ValueError, "Could not read image from path"): + _load_image_bytes(str(not_a_directory / "child.png")) + + def test_directory_path_raises_value_error(self): + with ( + tempfile.TemporaryDirectory() as temp_dir, + self.assertRaisesRegex(ValueError, "Could not read image from path"), + ): + _load_image_bytes(temp_dir) + + def test_overlong_path_raises_value_error(self): + # A raw base64 payload sent without the "data:" prefix falls through + # _resolve_media_item() as a "plain path" and ends up here. + with self.assertRaises(ValueError) as ctx: + _load_image_bytes("a" * 300) + self.assertIn("Could not read image from path", str(ctx.exception)) + self.assertLess(len(str(ctx.exception)), 200) + + def test_server_os_error_is_not_reclassified_as_client_error(self): + with ( + patch( + "builtins.open", + side_effect=OSError(errno.EMFILE, "too many open files"), + ), + self.assertRaises(OSError) as ctx, + ): + _load_image_bytes("/any/image.png") + self.assertEqual(ctx.exception.errno, errno.EMFILE) + + +class TestLoadAudioBytesIsClientError(CustomTestCase): + """_load_audio_bytes() (feature_extraction.py) -- same gap as the image loader + above, for the audio modality.""" + + def test_missing_local_path_raises_value_error(self): + with self.assertRaises(ValueError) as ctx: + _load_audio_bytes("/nonexistent/path/that/does/not/exist.wav") + self.assertIn("Could not read audio from path", str(ctx.exception)) + self.assertIsInstance(ctx.exception.__cause__, OSError) + + def test_path_component_that_is_not_a_directory_raises_value_error(self): + with tempfile.TemporaryDirectory() as temp_dir: + not_a_directory = Path(temp_dir) / "file" + not_a_directory.write_bytes(b"x") + with self.assertRaisesRegex(ValueError, "Could not read audio from path"): + _load_audio_bytes(str(not_a_directory / "child.wav")) + + def test_directory_path_raises_value_error(self): + with ( + tempfile.TemporaryDirectory() as temp_dir, + self.assertRaisesRegex(ValueError, "Could not read audio from path"), + ): + _load_audio_bytes(temp_dir) + + def test_overlong_path_raises_value_error(self): + with self.assertRaises(ValueError) as ctx: + _load_audio_bytes("a" * 300) + self.assertIn("Could not read audio from path", str(ctx.exception)) + self.assertLess(len(str(ctx.exception)), 200) + + def test_server_os_error_is_not_reclassified_as_client_error(self): + with ( + patch( + "builtins.open", + side_effect=OSError(errno.EMFILE, "too many open files"), + ), + self.assertRaises(OSError) as ctx, + ): + _load_audio_bytes("/any/audio.wav") + self.assertEqual(ctx.exception.errno, errno.EMFILE) + + +class TestDecodeAudioIsClientError(CustomTestCase): + """_decode_audio() (feature_extraction.py) -- undecodable audio bytes. + soundfile raises LibsndfileError, a RuntimeError subclass, so it must be + explicitly caught and reclassified.""" + + def test_undecodable_audio_bytes_raises_value_error(self): + with self.assertRaises(ValueError) as ctx: + _decode_audio(b"definitely not audio", sample_rate=16000) + self.assertIn("Could not decode audio", str(ctx.exception)) + self.assertIsInstance(ctx.exception.__cause__, sf.LibsndfileError) + + def test_valid_audio_still_decodes(self): + # Regression guard: a 1-second 440Hz tone should still decode without + # raising, and resampling (source rate == target rate here, so it's a + # no-op) must not be affected either. + sample_rate = 16000 + t = np.linspace(0, 1, sample_rate, endpoint=False) + tone = (0.1 * np.sin(2 * np.pi * 440 * t)).astype(np.float32) + buf = io.BytesIO() + sf.write(buf, tone, sample_rate, format="WAV") + + result = _decode_audio(buf.getvalue(), sample_rate=sample_rate) + self.assertEqual(result.shape[0], sample_rate) + + +if __name__ == "__main__": + unittest.main()