[Fix] glm4v: Clamp request video budgets to the deployment limit (#43871)

Co-authored-by: cipheraxat <cipheraxat@users.noreply.github.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Akshat Anand
2026-10-18 17:31:33 +08:00
committed by GitHub
co-authored by cipheraxat Claude Opus 5.5
parent 6c3db0f520
commit 5a930cfd88
5 changed files with 266 additions and 14 deletions
@@ -29,6 +29,7 @@ from sglang.srt.multimodal.encoder_preprocessing import (
)
from sglang.srt.multimodal.processors.glm4v import (
_glm_effective_presize_budget,
_merge_glm_video_configs,
glm_budget_kwargs,
glm_decode_frames_at,
glm_max_image_tokens_from_configs,
@@ -467,7 +468,11 @@ class EncoderPreprocessor:
try:
frames = await asyncio.get_running_loop().run_in_executor(
self.io_executor,
glm_decode_frames_at,
functools.partial(
glm_decode_frames_at,
# Full-clip count. local_indices is this rank's shard.
budget_frame_count=len(global_indices),
),
vr,
local_indices,
video_config,
@@ -525,9 +530,7 @@ class EncoderPreprocessor:
mm_items, video_configs = split_glm_video_items(mm_items)
defaults = glm_processor_video_config(self.video_processor)
defaults.update(self.vision_config.get("video", {}))
video_configs = [
{**defaults, **dict(config or {})} for config in video_configs
]
video_configs = _merge_glm_video_configs(defaults, video_configs)
futures, _ = self._submit_data_loading_tasks(
mm_items, [Modality.VIDEO] * len(mm_items)
@@ -606,10 +606,10 @@ class ChatCompletionMessageContentVideoURL(BaseModel):
url: str
max_dynamic_patch: Optional[int] = None
min_dynamic_patch: Optional[int] = None
fps: Optional[float] = None
max_frames: Optional[int] = None
max_tokens_per_frame: Optional[int] = None
max_image_tokens: Optional[int] = None
fps: Optional[float] = Field(default=None, gt=0, allow_inf_nan=False)
max_frames: Optional[int] = Field(default=None, gt=0)
max_tokens_per_frame: Optional[int] = Field(default=None, gt=0)
max_image_tokens: Optional[int] = Field(default=None, gt=0)
class ChatCompletionMessageContentAudioURL(BaseModel):
@@ -16,7 +16,7 @@ from sglang.srt.multimodal.processors.base_processor import (
from sglang.srt.multimodal.processors.base_processor import (
MultimodalSpecialTokens,
)
from sglang.srt.utils import GLM_MEDIA_CONFIG_KEYS
from sglang.srt.utils import GLM_MEDIA_CONFIG_KEYS, print_warning_once
from sglang.srt.utils.video_decoder import VideoDecoderWrapper
try:
@@ -34,6 +34,7 @@ GLM_VIDEO_DEFAULT_MAX_FRAMES = 2048
GLM_VIDEO_PATCH_SIZE = 14
GLM_VIDEO_MERGE_SIZE = 2
GLM_VIDEO_PATCH_EXPAND_FACTOR = 4
GLM_VIDEO_FRAME_PIXELS_PER_TOKEN = (GLM_VIDEO_PATCH_SIZE * GLM_VIDEO_MERGE_SIZE) ** 2
def _glm_video_metadata(total_num_frames, fps, duration, frames_indices):
@@ -116,6 +117,8 @@ def glm_processor_video_config(processor):
for key in GLM_MEDIA_CONFIG_KEYS
if (value := getattr(processor, key, None)) is not None
}
if (max_duration := getattr(processor, "max_duration", None) or 0) > 0:
config["_max_duration"] = max_duration
budget = _glm_processor_resize_budget(processor)
if budget is not None:
config["_presize_budget"] = budget
@@ -155,9 +158,72 @@ def _glm_effective_presize_budget(video_config, effective_max_image_tokens):
return config
def _glm_video_budget_ceiling(defaults, key):
if defaults.get(key) is not None:
return defaults[key]
if key == "fps":
return GLM_VIDEO_DEFAULT_FPS
if key == "max_frames":
# HF samplers without max_frames (GLM-4.1V/4.6V) stop at max_duration * fps.
if max_duration := defaults.get("_max_duration"):
return int(max_duration * defaults.get("fps", GLM_VIDEO_DEFAULT_FPS))
return GLM_VIDEO_DEFAULT_MAX_FRAMES
return None
def _clamp_glm_video_request_config(defaults, request):
"""Let request knobs lower the deployment budget, never raise it.
Omitted keys stay unset so an injected max_frames does not bypass the HF sampler.
"""
merged = {**defaults, **request}
if defaults.get("max_tokens_per_frame") is not None:
merged["_server_max_tokens_per_frame"] = True
for key in GLM_MEDIA_CONFIG_KEYS:
if key not in request:
continue
number = float if key == "fps" else int
try:
value = number(request[key])
valid = value > 0 and (number is int or math.isfinite(value))
except (TypeError, ValueError, OverflowError):
valid = False
if not valid:
raise ValueError(f"GLM video {key} must be a positive finite number")
ceiling = _glm_video_budget_ceiling(defaults, key)
if ceiling is not None and value > number(ceiling):
print_warning_once(
f"GLM video request {key} is clamped to the deployment limit "
f"{ceiling}; raise it with --mm-process-config."
)
value = number(ceiling)
merged[key] = value
return merged
def _glm_video_config_for_decode(video_config, frame_count):
"""Drop a request per-frame cap whose total pixels exceed the shared presize budget.
``frame_count`` is the full clip, not a DP shard. A server-configured cap stays.
"""
config = dict(video_config or {})
cap = config.get("max_tokens_per_frame")
budget = config.get("_presize_budget")
if cap is None or not budget or config.get("_server_max_tokens_per_frame"):
return config
# Per-frame resize counts 28x28 px per token; presize max_pixels spans all frames.
frame_pixels = int(cap) * GLM_VIDEO_FRAME_PIXELS_PER_TOKEN
if frame_pixels * int(frame_count) > budget["max_pixels"]:
config.pop("max_tokens_per_frame", None)
return config
def _merge_glm_video_configs(default_config, item_configs):
defaults = dict(default_config or {})
return [{**defaults, **dict(config or {})} for config in item_configs]
return [
_clamp_glm_video_request_config(defaults, dict(config or {}))
for config in item_configs
]
def _hf_sample_frame_indices(
@@ -250,7 +316,7 @@ def _resize_frames_to_max_tokens(frames, max_tokens_per_frame):
frames = torch.from_numpy(np.asarray(frames))
nchw = frames.permute(0, 3, 1, 2)
_, _, height, width = nchw.shape
pixels_per_token = (GLM_VIDEO_PATCH_SIZE * GLM_VIDEO_MERGE_SIZE) ** 2
pixels_per_token = GLM_VIDEO_FRAME_PIXELS_PER_TOKEN
factor = GLM_VIDEO_PATCH_SIZE * GLM_VIDEO_MERGE_SIZE * GLM_VIDEO_PATCH_EXPAND_FACTOR
max_pixels = max(
int(max_tokens_per_frame) * pixels_per_token,
@@ -388,11 +454,14 @@ def _pre_resize_frames_for_processor(
return nchw.permute(0, 2, 3, 1).contiguous()
def glm_decode_frames_at(vr, indices, video_config=None):
def glm_decode_frames_at(vr, indices, video_config=None, budget_frame_count=None):
"""Decode ``indices`` and resize under the shared token budget."""
indices = list(indices)
if not indices:
return None
video_config = video_config or {}
if budget_frame_count is None:
budget_frame_count = len(indices)
video_config = _glm_video_config_for_decode(video_config, budget_frame_count)
if hasattr(vr, "get_frames_as_tensor"):
frames = vr.get_frames_as_tensor(indices)
else: