[Fix] inkling: Return 400 instead of 500 for unfetchable/undecodable multimodal input (#40899)

Co-authored-by: apex-mochen <265981728+apex-mochen@users.noreply.github.com>
Co-authored-by: Po-Han Huang (NVIDIA) <53919306+nvpohanh@users.noreply.github.com>
Co-authored-by: apex-mochen <2756823972@qq.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
Co-authored-by: Po-Han Huang <pohanh@nvidia.com>
This commit is contained in:
hanahhh
2026-10-20 09:18:18 +08:00
committed by GitHub
co-authored by apex-mochen Po-Han Huang apex-mochen Claude Opus 5.5 Po-Han Huang
parent 693d333306
commit 71ee2dc950
5 changed files with 369 additions and 15 deletions
@@ -2,6 +2,7 @@
from __future__ import annotations
import errno
import io
import math
from dataclasses import dataclass
@@ -29,6 +30,10 @@ class InklingAudioEncoderParams:
audio_token_duration_s: float = 0.05
# Anything else (EMFILE, ENOSPC, EACCES, ...) is a server-side fault, not a bad path.
_EXPECTED_PATH_ERRNOS = {errno.ENOENT, errno.ENOTDIR, errno.EISDIR, errno.ENAMETOOLONG}
def _load_audio_bytes(audio) -> bytes:
"""Coerce a single audio input into raw file bytes for the encoder.
@@ -41,8 +46,19 @@ def _load_audio_bytes(audio) -> bytes:
return audio.read()
if isinstance(audio, str):
path = audio[len("file://") :] if audio.startswith("file://") else audio
with open(path, "rb") as f:
return f.read()
try:
with open(path, "rb") as f:
return f.read()
except OSError as e:
if e.errno not in _EXPECTED_PATH_ERRNOS:
raise
path_str = path
if len(path_str) > 100:
path_str = path_str[:100] + "..."
# str(e) re-embeds the full, untruncated path; use e.strerror instead.
raise ValueError(
f"Could not read audio from path {path_str}: {e.strerror}"
) from e
raise TypeError(
f"Unsupported audio input type for Inkling audio extractor: {type(audio)}"
)
@@ -56,9 +72,12 @@ def _to_exact_int(value: float, name: str, tolerance: float = 1e-6) -> int:
def _decode_audio(audio_bytes: bytes, sample_rate: int) -> torch.Tensor:
samples, src_sample_rate = sf.read(
io.BytesIO(audio_bytes), dtype="float32", always_2d=True
)
try:
samples, src_sample_rate = sf.read(
io.BytesIO(audio_bytes), dtype="float32", always_2d=True
)
except sf.LibsndfileError as e:
raise ValueError(f"Could not decode audio: {e}") from e
mono = samples.mean(axis=1)
if src_sample_rate != sample_rate:
mono = _resample(mono, src_sample_rate, sample_rate)
@@ -2,6 +2,7 @@
from __future__ import annotations
import errno
import io
import math
from typing import List, Optional, Union
@@ -17,6 +18,9 @@ IMAGE_STD = np.array([0.26862954, 0.2613026, 0.2757771], dtype=np.float32)
PAD_RAW_VALUE = np.float32(-1.0 / 255.0)
PAD_NORM = (np.full((3,), PAD_RAW_VALUE, dtype=np.float32) - IMAGE_MEAN) / IMAGE_STD
# Anything else (EMFILE, ENOSPC, EACCES, ...) is a server-side fault, not a bad path.
_EXPECTED_PATH_ERRNOS = {errno.ENOENT, errno.ENOTDIR, errno.EISDIR, errno.ENAMETOOLONG}
def _validate_image_rescale(
rescale_image_frac: Optional[float],
@@ -86,8 +90,19 @@ def _load_image_bytes(image) -> bytes:
"upstream (e.g. via SGLang load_mm_data) before preprocessing."
)
path = image[len("file://") :] if image.startswith("file://") else image
with open(path, "rb") as f:
return f.read()
try:
with open(path, "rb") as f:
return f.read()
except OSError as e:
if e.errno not in _EXPECTED_PATH_ERRNOS:
raise
path_str = path
if len(path_str) > 100:
path_str = path_str[:100] + "..."
# str(e) re-embeds the full, untruncated path; use e.strerror instead.
raise ValueError(
f"Could not read image from path {path_str}: {e.strerror}"
) from e
from PIL import Image
@@ -153,7 +168,10 @@ def _encode_image_bytes(
from PIL import Image
image = Image.open(io.BytesIO(image_bytes)).convert("RGB")
try:
image = Image.open(io.BytesIO(image_bytes)).convert("RGB")
except (OSError, SyntaxError) as e:
raise ValueError(f"Could not decode image: {e}") from e
scaled_size = _scaled_image_dimensions(
*image.size,
rescale_image_frac=rescale_image_frac,
@@ -34,7 +34,11 @@ _pil_pool = ThreadPoolExecutor(max_workers=8)
def _pil_decode(raw: bytes) -> np.ndarray:
return np.ascontiguousarray(np.array(Image.open(io.BytesIO(raw)).convert("RGB")))
try:
image = Image.open(io.BytesIO(raw)).convert("RGB")
except (OSError, SyntaxError) as e:
raise ValueError(f"Could not decode image: {e}") from e
return np.ascontiguousarray(np.array(image))
class InklingRustImageProcessor(BaseImageProcessor):
@@ -52,7 +52,7 @@ from sglang.srt.parser.inkling_tokenizer import IMAGE_TOKEN_ID as INKLING_IMAGE_
from sglang.srt.parser.inkling_tokenizer import (
INKLING_SPECIAL_TOKEN_IDS,
)
from sglang.srt.utils.common import download_remote_media
from sglang.srt.utils.common import CLIENT_MEDIA_EXCEPTIONS, download_remote_media
logger = logging.getLogger(__name__)
@@ -78,11 +78,21 @@ def _resolve_media_item(item):
url = getattr(item, "url")
if not isinstance(url, str):
return item
if url.startswith("data:"):
header, _, payload = url.partition(",")
return base64.b64decode(payload) if ";base64" in header else payload.encode()
if url.startswith(("http://", "https://")):
return download_remote_media(url, timeout=30)
if not url.strip():
raise ValueError("empty media URL")
try:
if url.startswith("data:"):
header, _, payload = url.partition(",")
return (
base64.b64decode(payload) if ";base64" in header else payload.encode()
)
if url.startswith(("http://", "https://")):
return download_remote_media(url, timeout=30)
except CLIENT_MEDIA_EXCEPTIONS as e:
url_str = url
if len(url_str) > 100:
url_str = url_str[:100] + "..."
raise ValueError(f"Error while resolving media {url_str}: {e}") from e
return url # plain path / file:// -> handled by the per-modality byte loader