[Fix] inkling: Return 400 instead of 500 for unfetchable/undecodable multimodal input (#40899)
Co-authored-by: apex-mochen <265981728+apex-mochen@users.noreply.github.com> Co-authored-by: Po-Han Huang (NVIDIA) <53919306+nvpohanh@users.noreply.github.com> Co-authored-by: apex-mochen <2756823972@qq.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: Po-Han Huang <pohanh@nvidia.com>
This commit is contained in:
co-authored by
apex-mochen
Po-Han Huang
apex-mochen
Claude Opus 5.5
Po-Han Huang
parent
693d333306
commit
71ee2dc950
@@ -2,6 +2,7 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import errno
|
||||
import io
|
||||
import math
|
||||
from dataclasses import dataclass
|
||||
@@ -29,6 +30,10 @@ class InklingAudioEncoderParams:
|
||||
audio_token_duration_s: float = 0.05
|
||||
|
||||
|
||||
# Anything else (EMFILE, ENOSPC, EACCES, ...) is a server-side fault, not a bad path.
|
||||
_EXPECTED_PATH_ERRNOS = {errno.ENOENT, errno.ENOTDIR, errno.EISDIR, errno.ENAMETOOLONG}
|
||||
|
||||
|
||||
def _load_audio_bytes(audio) -> bytes:
|
||||
"""Coerce a single audio input into raw file bytes for the encoder.
|
||||
|
||||
@@ -41,8 +46,19 @@ def _load_audio_bytes(audio) -> bytes:
|
||||
return audio.read()
|
||||
if isinstance(audio, str):
|
||||
path = audio[len("file://") :] if audio.startswith("file://") else audio
|
||||
with open(path, "rb") as f:
|
||||
return f.read()
|
||||
try:
|
||||
with open(path, "rb") as f:
|
||||
return f.read()
|
||||
except OSError as e:
|
||||
if e.errno not in _EXPECTED_PATH_ERRNOS:
|
||||
raise
|
||||
path_str = path
|
||||
if len(path_str) > 100:
|
||||
path_str = path_str[:100] + "..."
|
||||
# str(e) re-embeds the full, untruncated path; use e.strerror instead.
|
||||
raise ValueError(
|
||||
f"Could not read audio from path {path_str}: {e.strerror}"
|
||||
) from e
|
||||
raise TypeError(
|
||||
f"Unsupported audio input type for Inkling audio extractor: {type(audio)}"
|
||||
)
|
||||
@@ -56,9 +72,12 @@ def _to_exact_int(value: float, name: str, tolerance: float = 1e-6) -> int:
|
||||
|
||||
|
||||
def _decode_audio(audio_bytes: bytes, sample_rate: int) -> torch.Tensor:
|
||||
samples, src_sample_rate = sf.read(
|
||||
io.BytesIO(audio_bytes), dtype="float32", always_2d=True
|
||||
)
|
||||
try:
|
||||
samples, src_sample_rate = sf.read(
|
||||
io.BytesIO(audio_bytes), dtype="float32", always_2d=True
|
||||
)
|
||||
except sf.LibsndfileError as e:
|
||||
raise ValueError(f"Could not decode audio: {e}") from e
|
||||
mono = samples.mean(axis=1)
|
||||
if src_sample_rate != sample_rate:
|
||||
mono = _resample(mono, src_sample_rate, sample_rate)
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import errno
|
||||
import io
|
||||
import math
|
||||
from typing import List, Optional, Union
|
||||
@@ -17,6 +18,9 @@ IMAGE_STD = np.array([0.26862954, 0.2613026, 0.2757771], dtype=np.float32)
|
||||
PAD_RAW_VALUE = np.float32(-1.0 / 255.0)
|
||||
PAD_NORM = (np.full((3,), PAD_RAW_VALUE, dtype=np.float32) - IMAGE_MEAN) / IMAGE_STD
|
||||
|
||||
# Anything else (EMFILE, ENOSPC, EACCES, ...) is a server-side fault, not a bad path.
|
||||
_EXPECTED_PATH_ERRNOS = {errno.ENOENT, errno.ENOTDIR, errno.EISDIR, errno.ENAMETOOLONG}
|
||||
|
||||
|
||||
def _validate_image_rescale(
|
||||
rescale_image_frac: Optional[float],
|
||||
@@ -86,8 +90,19 @@ def _load_image_bytes(image) -> bytes:
|
||||
"upstream (e.g. via SGLang load_mm_data) before preprocessing."
|
||||
)
|
||||
path = image[len("file://") :] if image.startswith("file://") else image
|
||||
with open(path, "rb") as f:
|
||||
return f.read()
|
||||
try:
|
||||
with open(path, "rb") as f:
|
||||
return f.read()
|
||||
except OSError as e:
|
||||
if e.errno not in _EXPECTED_PATH_ERRNOS:
|
||||
raise
|
||||
path_str = path
|
||||
if len(path_str) > 100:
|
||||
path_str = path_str[:100] + "..."
|
||||
# str(e) re-embeds the full, untruncated path; use e.strerror instead.
|
||||
raise ValueError(
|
||||
f"Could not read image from path {path_str}: {e.strerror}"
|
||||
) from e
|
||||
|
||||
from PIL import Image
|
||||
|
||||
@@ -153,7 +168,10 @@ def _encode_image_bytes(
|
||||
|
||||
from PIL import Image
|
||||
|
||||
image = Image.open(io.BytesIO(image_bytes)).convert("RGB")
|
||||
try:
|
||||
image = Image.open(io.BytesIO(image_bytes)).convert("RGB")
|
||||
except (OSError, SyntaxError) as e:
|
||||
raise ValueError(f"Could not decode image: {e}") from e
|
||||
scaled_size = _scaled_image_dimensions(
|
||||
*image.size,
|
||||
rescale_image_frac=rescale_image_frac,
|
||||
|
||||
@@ -34,7 +34,11 @@ _pil_pool = ThreadPoolExecutor(max_workers=8)
|
||||
|
||||
|
||||
def _pil_decode(raw: bytes) -> np.ndarray:
|
||||
return np.ascontiguousarray(np.array(Image.open(io.BytesIO(raw)).convert("RGB")))
|
||||
try:
|
||||
image = Image.open(io.BytesIO(raw)).convert("RGB")
|
||||
except (OSError, SyntaxError) as e:
|
||||
raise ValueError(f"Could not decode image: {e}") from e
|
||||
return np.ascontiguousarray(np.array(image))
|
||||
|
||||
|
||||
class InklingRustImageProcessor(BaseImageProcessor):
|
||||
|
||||
@@ -52,7 +52,7 @@ from sglang.srt.parser.inkling_tokenizer import IMAGE_TOKEN_ID as INKLING_IMAGE_
|
||||
from sglang.srt.parser.inkling_tokenizer import (
|
||||
INKLING_SPECIAL_TOKEN_IDS,
|
||||
)
|
||||
from sglang.srt.utils.common import download_remote_media
|
||||
from sglang.srt.utils.common import CLIENT_MEDIA_EXCEPTIONS, download_remote_media
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -78,11 +78,21 @@ def _resolve_media_item(item):
|
||||
url = getattr(item, "url")
|
||||
if not isinstance(url, str):
|
||||
return item
|
||||
if url.startswith("data:"):
|
||||
header, _, payload = url.partition(",")
|
||||
return base64.b64decode(payload) if ";base64" in header else payload.encode()
|
||||
if url.startswith(("http://", "https://")):
|
||||
return download_remote_media(url, timeout=30)
|
||||
if not url.strip():
|
||||
raise ValueError("empty media URL")
|
||||
try:
|
||||
if url.startswith("data:"):
|
||||
header, _, payload = url.partition(",")
|
||||
return (
|
||||
base64.b64decode(payload) if ";base64" in header else payload.encode()
|
||||
)
|
||||
if url.startswith(("http://", "https://")):
|
||||
return download_remote_media(url, timeout=30)
|
||||
except CLIENT_MEDIA_EXCEPTIONS as e:
|
||||
url_str = url
|
||||
if len(url_str) > 100:
|
||||
url_str = url_str[:100] + "..."
|
||||
raise ValueError(f"Error while resolving media {url_str}: {e}") from e
|
||||
return url # plain path / file:// -> handled by the per-modality byte loader
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user