diff --git a/docs/docs/supported-models/multimodal_language_models.mdx b/docs/docs/supported-models/multimodal_language_models.mdx index f7e5010a3e..8680ed087b 100644 --- a/docs/docs/supported-models/multimodal_language_models.mdx +++ b/docs/docs/supported-models/multimodal_language_models.mdx @@ -422,6 +422,8 @@ Use this option when you have sufficient GPU memory and want to minimize latency This can reduce GPU memory usage, improve inference speed, and help to avoid OOM, but may impact model performance, thus set a proper value based on your specific use case. The config entries are passed as `images_kwargs`, `videos_kwargs`, and `audio_kwargs` to the HuggingFace processor, so each modality's settings are kept separate and do not collide. Refer to the HuggingFace documentation for your model's processor to understand the available parameters. +For GLM-V models, the per-request `video_url` fields `fps`, `max_frames`, `max_tokens_per_frame`, and `max_image_tokens` can only tighten the deployment's video budget. Larger `fps`, `max_frames`, and `max_image_tokens` values are clamped to the deployment setting, and a `max_tokens_per_frame` that would exceed the video token budget is ignored. Raise the limits with `--mm-process-config` instead. + ### Bidirectional Attention in Multimodal Model Serving **Note for serving the Gemma-3 multimodal model**: diff --git a/python/sglang/srt/disaggregation/encoder/preprocessor.py b/python/sglang/srt/disaggregation/encoder/preprocessor.py index 5b84789f44..3af6791a3f 100644 --- a/python/sglang/srt/disaggregation/encoder/preprocessor.py +++ b/python/sglang/srt/disaggregation/encoder/preprocessor.py @@ -29,6 +29,7 @@ from sglang.srt.multimodal.encoder_preprocessing import ( ) from sglang.srt.multimodal.processors.glm4v import ( _glm_effective_presize_budget, + _merge_glm_video_configs, glm_budget_kwargs, glm_decode_frames_at, glm_max_image_tokens_from_configs, @@ -467,7 +468,11 @@ class EncoderPreprocessor: try: frames = await asyncio.get_running_loop().run_in_executor( self.io_executor, - glm_decode_frames_at, + functools.partial( + glm_decode_frames_at, + # Full-clip count. local_indices is this rank's shard. + budget_frame_count=len(global_indices), + ), vr, local_indices, video_config, @@ -525,9 +530,7 @@ class EncoderPreprocessor: mm_items, video_configs = split_glm_video_items(mm_items) defaults = glm_processor_video_config(self.video_processor) defaults.update(self.vision_config.get("video", {})) - video_configs = [ - {**defaults, **dict(config or {})} for config in video_configs - ] + video_configs = _merge_glm_video_configs(defaults, video_configs) futures, _ = self._submit_data_loading_tasks( mm_items, [Modality.VIDEO] * len(mm_items) diff --git a/python/sglang/srt/entrypoints/openai/protocol.py b/python/sglang/srt/entrypoints/openai/protocol.py index 36c1a41f67..641ccde476 100644 --- a/python/sglang/srt/entrypoints/openai/protocol.py +++ b/python/sglang/srt/entrypoints/openai/protocol.py @@ -606,10 +606,10 @@ class ChatCompletionMessageContentVideoURL(BaseModel): url: str max_dynamic_patch: Optional[int] = None min_dynamic_patch: Optional[int] = None - fps: Optional[float] = None - max_frames: Optional[int] = None - max_tokens_per_frame: Optional[int] = None - max_image_tokens: Optional[int] = None + fps: Optional[float] = Field(default=None, gt=0, allow_inf_nan=False) + max_frames: Optional[int] = Field(default=None, gt=0) + max_tokens_per_frame: Optional[int] = Field(default=None, gt=0) + max_image_tokens: Optional[int] = Field(default=None, gt=0) class ChatCompletionMessageContentAudioURL(BaseModel): diff --git a/python/sglang/srt/multimodal/processors/glm4v.py b/python/sglang/srt/multimodal/processors/glm4v.py index 105814e25a..aee01de5db 100644 --- a/python/sglang/srt/multimodal/processors/glm4v.py +++ b/python/sglang/srt/multimodal/processors/glm4v.py @@ -16,7 +16,7 @@ from sglang.srt.multimodal.processors.base_processor import ( from sglang.srt.multimodal.processors.base_processor import ( MultimodalSpecialTokens, ) -from sglang.srt.utils import GLM_MEDIA_CONFIG_KEYS +from sglang.srt.utils import GLM_MEDIA_CONFIG_KEYS, print_warning_once from sglang.srt.utils.video_decoder import VideoDecoderWrapper try: @@ -34,6 +34,7 @@ GLM_VIDEO_DEFAULT_MAX_FRAMES = 2048 GLM_VIDEO_PATCH_SIZE = 14 GLM_VIDEO_MERGE_SIZE = 2 GLM_VIDEO_PATCH_EXPAND_FACTOR = 4 +GLM_VIDEO_FRAME_PIXELS_PER_TOKEN = (GLM_VIDEO_PATCH_SIZE * GLM_VIDEO_MERGE_SIZE) ** 2 def _glm_video_metadata(total_num_frames, fps, duration, frames_indices): @@ -116,6 +117,8 @@ def glm_processor_video_config(processor): for key in GLM_MEDIA_CONFIG_KEYS if (value := getattr(processor, key, None)) is not None } + if (max_duration := getattr(processor, "max_duration", None) or 0) > 0: + config["_max_duration"] = max_duration budget = _glm_processor_resize_budget(processor) if budget is not None: config["_presize_budget"] = budget @@ -155,9 +158,72 @@ def _glm_effective_presize_budget(video_config, effective_max_image_tokens): return config +def _glm_video_budget_ceiling(defaults, key): + if defaults.get(key) is not None: + return defaults[key] + if key == "fps": + return GLM_VIDEO_DEFAULT_FPS + if key == "max_frames": + # HF samplers without max_frames (GLM-4.1V/4.6V) stop at max_duration * fps. + if max_duration := defaults.get("_max_duration"): + return int(max_duration * defaults.get("fps", GLM_VIDEO_DEFAULT_FPS)) + return GLM_VIDEO_DEFAULT_MAX_FRAMES + return None + + +def _clamp_glm_video_request_config(defaults, request): + """Let request knobs lower the deployment budget, never raise it. + + Omitted keys stay unset so an injected max_frames does not bypass the HF sampler. + """ + merged = {**defaults, **request} + if defaults.get("max_tokens_per_frame") is not None: + merged["_server_max_tokens_per_frame"] = True + for key in GLM_MEDIA_CONFIG_KEYS: + if key not in request: + continue + number = float if key == "fps" else int + try: + value = number(request[key]) + valid = value > 0 and (number is int or math.isfinite(value)) + except (TypeError, ValueError, OverflowError): + valid = False + if not valid: + raise ValueError(f"GLM video {key} must be a positive finite number") + ceiling = _glm_video_budget_ceiling(defaults, key) + if ceiling is not None and value > number(ceiling): + print_warning_once( + f"GLM video request {key} is clamped to the deployment limit " + f"{ceiling}; raise it with --mm-process-config." + ) + value = number(ceiling) + merged[key] = value + return merged + + +def _glm_video_config_for_decode(video_config, frame_count): + """Drop a request per-frame cap whose total pixels exceed the shared presize budget. + + ``frame_count`` is the full clip, not a DP shard. A server-configured cap stays. + """ + config = dict(video_config or {}) + cap = config.get("max_tokens_per_frame") + budget = config.get("_presize_budget") + if cap is None or not budget or config.get("_server_max_tokens_per_frame"): + return config + # Per-frame resize counts 28x28 px per token; presize max_pixels spans all frames. + frame_pixels = int(cap) * GLM_VIDEO_FRAME_PIXELS_PER_TOKEN + if frame_pixels * int(frame_count) > budget["max_pixels"]: + config.pop("max_tokens_per_frame", None) + return config + + def _merge_glm_video_configs(default_config, item_configs): defaults = dict(default_config or {}) - return [{**defaults, **dict(config or {})} for config in item_configs] + return [ + _clamp_glm_video_request_config(defaults, dict(config or {})) + for config in item_configs + ] def _hf_sample_frame_indices( @@ -250,7 +316,7 @@ def _resize_frames_to_max_tokens(frames, max_tokens_per_frame): frames = torch.from_numpy(np.asarray(frames)) nchw = frames.permute(0, 3, 1, 2) _, _, height, width = nchw.shape - pixels_per_token = (GLM_VIDEO_PATCH_SIZE * GLM_VIDEO_MERGE_SIZE) ** 2 + pixels_per_token = GLM_VIDEO_FRAME_PIXELS_PER_TOKEN factor = GLM_VIDEO_PATCH_SIZE * GLM_VIDEO_MERGE_SIZE * GLM_VIDEO_PATCH_EXPAND_FACTOR max_pixels = max( int(max_tokens_per_frame) * pixels_per_token, @@ -388,11 +454,14 @@ def _pre_resize_frames_for_processor( return nchw.permute(0, 2, 3, 1).contiguous() -def glm_decode_frames_at(vr, indices, video_config=None): +def glm_decode_frames_at(vr, indices, video_config=None, budget_frame_count=None): + """Decode ``indices`` and resize under the shared token budget.""" indices = list(indices) if not indices: return None - video_config = video_config or {} + if budget_frame_count is None: + budget_frame_count = len(indices) + video_config = _glm_video_config_for_decode(video_config, budget_frame_count) if hasattr(vr, "get_frames_as_tensor"): frames = vr.get_frames_as_tensor(indices) else: diff --git a/test/registered/unit/multimodal/test_glm4v_mixed_offsets.py b/test/registered/unit/multimodal/test_glm4v_mixed_offsets.py index 1384c12c16..54a0b8a915 100644 --- a/test/registered/unit/multimodal/test_glm4v_mixed_offsets.py +++ b/test/registered/unit/multimodal/test_glm4v_mixed_offsets.py @@ -1,9 +1,19 @@ +import numpy as np import pytest import torch +from pydantic import ValidationError +from sglang.srt.entrypoints.openai.protocol import ChatCompletionMessageContentVideoURL from sglang.srt.managers.schedule_batch import Modality +from sglang.srt.multimodal.processors import glm4v as glm4v_module from sglang.srt.multimodal.processors.base_processor import MultimodalSpecialTokens -from sglang.srt.multimodal.processors.glm4v import Glm4vImageProcessor +from sglang.srt.multimodal.processors.glm4v import ( + Glm4vImageProcessor, + _merge_glm_video_configs, + glm_decode_frames_at, + glm_processor_video_config, + glm_sample_and_decode_sync, +) from sglang.test.ci.ci_register import register_cpu_ci register_cpu_ci(est_time=9, suite="base-a-test-cpu") @@ -97,5 +107,173 @@ def test_glm4v_uses_default_offsets_when_token_ids_are_distinct(): ] +class _Glm5NextLikeVideoProcessor: + fps = 2 + max_frames = 2048 + max_duration = 0 + max_image_tokens = 16384 + patch_size = 14 + merge_size = 2 + temporal_patch_size = 2 + + +class _Glm4vLikeVideoProcessor: + """GLM-4.1V/4.6V shape: no max_frames, HF sampler stops at max_duration * fps.""" + + fps = 2 + max_duration = 300 + + def sample_frames(self, metadata, fps=None): + duration = min(metadata.duration, self.max_duration) + count = int(duration * (fps or self.fps)) + return np.linspace(0, metadata.total_num_frames - 1, count, dtype=int) + + +class _FakeVideoReader: + avg_fps = 30.0 + + def __init__(self, seconds): + self._num_frames = int(seconds * self.avg_fps) + + def __len__(self): + return self._num_frames + + def get_frames_at(self, indices): + return np.zeros((len(indices), 2, 2, 3), dtype=np.uint8) + + +def _video_defaults(processor, **server_video_config): + return {**glm_processor_video_config(processor), **server_video_config} + + +def _merge_one(defaults, request): + return _merge_glm_video_configs(defaults, [request])[0] + + +def test_glm_video_merge_does_not_inject_omitted_max_frames(): + """Any max_frames bypasses the HF sampler, so an fps-only request must not gain one.""" + defaults = _video_defaults(_Glm4vLikeVideoProcessor()) + assert _merge_one(defaults, {}) == defaults + assert "max_frames" not in _merge_one(defaults, {"fps": 1}) + + +@pytest.mark.parametrize( + "server, request_config, key, expected", + [ + ({}, {"fps": 60}, "fps", 2.0), + ({}, {"max_frames": 1_000_000}, "max_frames", 2048), + ({}, {"max_frames": 10**400}, "max_frames", 2048), + ({}, {"max_image_tokens": 10**9}, "max_image_tokens", 16384), + ({"max_frames": 4096}, {"max_frames": 10**6}, "max_frames", 4096), + ( + {"max_tokens_per_frame": 512}, + {"max_tokens_per_frame": 10**6}, + "max_tokens_per_frame", + 512, + ), + ], +) +def test_glm_video_merge_clamps_request_to_deployment_limit( + server, request_config, key, expected +): + defaults = _video_defaults(_Glm5NextLikeVideoProcessor(), **server) + assert _merge_one(defaults, request_config)[key] == expected + + +def test_glm_video_merge_keeps_tighter_request(): + defaults = _video_defaults(_Glm5NextLikeVideoProcessor()) + request = { + "fps": 1, + "max_frames": 8, + "max_image_tokens": 1000, + "max_tokens_per_frame": 100, + } + merged = _merge_one(defaults, request) + assert {key: merged[key] for key in request} == request + + +@pytest.mark.parametrize("server", [{}, {"fps": 3}]) +def test_glm4v_request_cannot_sample_more_frames_than_default(server): + """A request max_frames switches to the fallback sampler; it must not exceed the HF default count.""" + processor = _Glm4vLikeVideoProcessor() + defaults = _video_defaults(processor, **server) + video = _FakeVideoReader(seconds=3600) + + def frame_count(request): + frames, _ = glm_sample_and_decode_sync( + video, _merge_one(defaults, request), processor + ) + return len(frames) + + default_count = frame_count({}) + assert frame_count({"fps": 60, "max_frames": 1_000_000}) <= default_count + + +def _patch_glm_resizers(monkeypatch): + monkeypatch.setattr( + glm4v_module, + "_resize_frames_to_max_tokens", + lambda frames, max_tokens_per_frame: ("per_frame", max_tokens_per_frame), + ) + monkeypatch.setattr( + glm4v_module, + "_pre_resize_frames_for_processor", + lambda frames, **budget: ("presize",), + ) + + +def test_glm_decode_drops_request_per_frame_cap_over_pixel_budget(monkeypatch): + """A request per-frame cap is kept only while its total pixels fit the shared budget. + + Per-frame resize counts 784 px per token and presize counts 1568, so 16384 tokens + x 2 frames fits a 16384-token video budget exactly. + """ + _patch_glm_resizers(monkeypatch) + defaults = _video_defaults(_Glm5NextLikeVideoProcessor()) + config = _merge_one(defaults, {"max_tokens_per_frame": 16384}) + video = _FakeVideoReader(seconds=1) + + assert glm_decode_frames_at(video, [0, 1], config) == ("per_frame", 16384) + assert glm_decode_frames_at(video, [0, 1, 2], config) == ("presize",) + # A DP shard decides on the full clip length, not its own slice. + assert glm_decode_frames_at(video, [0], config, budget_frame_count=3) == ( + "presize", + ) + + +def test_glm_decode_keeps_server_per_frame_mode(monkeypatch): + _patch_glm_resizers(monkeypatch) + defaults = _video_defaults(_Glm5NextLikeVideoProcessor(), max_tokens_per_frame=512) + config = _merge_one(defaults, {"max_tokens_per_frame": 100}) + frames = glm_decode_frames_at(_FakeVideoReader(seconds=1), range(200), config) + assert frames == ("per_frame", 100) + + +@pytest.mark.parametrize( + "request_config", + [ + {"fps": 0}, + {"fps": "fast"}, + {"fps": float("inf")}, + {"fps": float("nan")}, + {"max_frames": -1}, + {"max_frames": float("inf")}, + ], +) +def test_glm_video_merge_rejects_invalid_knobs(request_config): + (key,) = request_config + with pytest.raises(ValueError, match=key): + _merge_glm_video_configs({}, [request_config]) + + +@pytest.mark.parametrize( + "field, value", + [("fps", 0), ("fps", float("inf")), ("max_frames", 0), ("max_image_tokens", -1)], +) +def test_video_url_rejects_non_positive_knobs(field, value): + with pytest.raises(ValidationError): + ChatCompletionMessageContentVideoURL(url="video.mp4", **{field: value}) + + if __name__ == "__main__": raise SystemExit(pytest.main([__file__, "-v"]))