[Feat] MiniCPM-V-4.7: add model support (#41993)

Co-authored-by: Xinyuan Tong <115166877+JustinTong0323@users.noreply.github.com>
Co-authored-by: Xinyuan Tong <xinyuantong.cs@gmail.com>
This commit is contained in:
tc-mb
2026-10-16 13:24:48 +08:00
committed by GitHub
co-authored by Xinyuan Tong Xinyuan Tong
parent a2981ab8ea
commit 2a08e261ec
18 changed files with 714 additions and 74 deletions
@@ -10,6 +10,7 @@ from sglang.srt.runtime_context import get_platform
@_register_for("MiniCPMV4_6ForConditionalGeneration")
@_register_for("MiniCPMV4_7ForConditionalGeneration")
def _minicpm_v4_6_overrides(server_args: Any, hf_config: Any) -> dict:
cfg = resolving_view(server_args)
if get_platform().is_sm100 and cfg.attention_backend is None:
@@ -362,6 +362,7 @@ _MAMBA_RADIX_CACHE_ARCHS = frozenset(
"Qwen3_5ForCausalLM",
"Qwen4ExpForConditionalGeneration",
"MiniCPMV4_6ForConditionalGeneration",
"MiniCPMV4_7ForConditionalGeneration",
"NemotronHForCausalLM",
"NemotronHPuzzleForCausalLM",
"NemotronH_Omni_Reasoning_V3",
@@ -390,6 +391,7 @@ _MAMBA_EXTRA_BUFFER_ARCHS = frozenset(
"Qwen4ExpForConditionalGeneration",
"InternS2PreviewForConditionalGeneration",
"MiniCPMV4_6ForConditionalGeneration",
"MiniCPMV4_7ForConditionalGeneration",
"BailingMoeV2_5ForCausalLM",
"BailingMoeV3ForCausalLM",
"BailingMoeV3VLForConditionalGeneration",
+2
View File
@@ -56,6 +56,7 @@ from sglang.srt.configs.mamba import FalconMambaConfig, MambaConfig
from sglang.srt.configs.mamba2 import Mamba2Config
from sglang.srt.configs.minicpm import MiniCPMHybridConfig
from sglang.srt.configs.minicpmv4_6 import MiniCPMV4_6Config, MiniCPMV4_6VisionConfig
from sglang.srt.configs.minicpmv4_7 import MiniCPMV4_7Config
from sglang.srt.configs.minimax_vl import MiniMaxM3VLConfig
from sglang.srt.configs.muse_glimmer import (
MuseGlimmerAssistantConfig,
@@ -150,6 +151,7 @@ __all__ = [
"LocateAnythingConfig",
"MiniCPMV4_6Config",
"MiniCPMV4_6VisionConfig",
"MiniCPMV4_7Config",
"NemotronHConfig",
"NemotronHPuzzleConfig",
"NemotronH_Nano_VL_V2_Config",
+9 -7
View File
@@ -17,7 +17,7 @@ from typing import Any, Dict, Optional, Union
from transformers import AutoConfig, PretrainedConfig
from transformers.models.auto import CONFIG_MAPPING
from sglang.srt.configs.qwen3_5 import Qwen3_5TextConfig
from sglang.srt.configs.qwen3_5 import Qwen3_5MoeTextConfig, Qwen3_5TextConfig
class MiniCPMV4_6VisionConfig(PretrainedConfig):
@@ -54,15 +54,17 @@ class MiniCPMV4_6VisionConfig(PretrainedConfig):
def _resolve_text_config_class(model_type: Optional[str]) -> type:
"""``model_type`` -> registered config class. sglang's ``Qwen3_5TextConfig``
wins over the stock entry when both exist (it carries ``layers_block_type``
etc. that the model code reads); ``AutoConfig.register`` doesn't replace
existing entries so we have to short-circuit here. Note that
"""``model_type`` -> registered config class. sglang's Qwen3.5 dense / MoE
text configs win over the stock entries when both exist (they carry
``layers_block_type`` etc. that the model code reads);
``AutoConfig.register`` doesn't replace existing entries so we have to
short-circuit here. Note that
``CONFIG_MAPPING.get`` returns ``None`` even on hit — go through
``__getitem__`` to trigger the lazy class import.
"""
if model_type == Qwen3_5TextConfig.model_type:
return Qwen3_5TextConfig
for cls in (Qwen3_5TextConfig, Qwen3_5MoeTextConfig):
if model_type == cls.model_type:
return cls
if model_type and model_type in CONFIG_MAPPING:
return CONFIG_MAPPING[model_type]
raise KeyError(f"Unknown text_config model_type: {model_type!r}")
+22
View File
@@ -0,0 +1,22 @@
# Copyright 2026 The SGLang team.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
from typing import Any
from sglang.srt.configs.minicpmv4_6 import MiniCPMV4_6Config
class MiniCPMV4_7Config(MiniCPMV4_6Config):
model_type = "minicpmv4_7"
# transformers 5 wraps config subclasses in ``@dataclass(kw_only=True)``;
# its generated ``__init__`` would skip ``MiniCPMV4_6Config.__init__``.
def __init__(self, **kwargs: Any) -> None:
super().__init__(**kwargs)
__all__ = ["MiniCPMV4_7Config"]
@@ -407,6 +407,9 @@ class CompletionRequest(PDRoutingFields):
)
images_config: Optional[Dict] = None
# Per-request processor overrides; only MiniCPM-V 4.7 reads them (downsample_mode,
# max_slice_nums), flattening images_/videos_kwargs into one request-wide profile.
mm_processor_kwargs: Optional[Dict] = None
# Deprecated: use routed_dp_rank instead
data_parallel_rank: Optional[int] = None
@@ -966,6 +969,9 @@ class ChatCompletionRequest(PDRoutingFields):
images_config: Optional[Dict] = None
video_config: Optional[Dict] = None
# Per-request processor overrides; only MiniCPM-V 4.7 reads them (downsample_mode,
# max_slice_nums), flattening images_/videos_kwargs into one request-wide profile.
mm_processor_kwargs: Optional[Dict] = None
# Custom logit processor for advanced sampling control
custom_logit_processor: Optional[Union[List[Optional[str]], str]] = None
@@ -1328,6 +1328,7 @@ class OpenAIServingChat(OpenAIServingBase):
custom_logit_processor=request.custom_logit_processor,
images_config=getattr(request, "images_config", None),
video_config=_build_video_config(request),
mm_processor_kwargs=request.mm_processor_kwargs,
image_max_dynamic_patch=img_max_dynamic_patch,
video_max_dynamic_patch=vid_max_dynamic_patch,
max_dynamic_patch=getattr(request, "max_dynamic_patch", None),
@@ -140,6 +140,7 @@ class OpenAIServingCompletion(OpenAIServingBase):
custom_labels=custom_labels,
custom_logit_processor=request.custom_logit_processor,
images_config=getattr(request, "images_config", None),
mm_processor_kwargs=request.mm_processor_kwargs,
)
return adapted_request, request
+2
View File
@@ -355,6 +355,7 @@ class GenerateReqInput:
# For Unlimited-OCR
images_config: Optional[dict] = None
mm_processor_kwargs: Optional[dict] = None
# Pre-computed delimiter indices for multi-item scoring.
# Batch-level: List[List[int]] (one per request). After __getitem__: List[int].
@@ -1009,6 +1010,7 @@ class GenerateReqInput:
if self.custom_logit_processor is not None
else None
),
mm_processor_kwargs=self.mm_processor_kwargs,
positional_embed_overrides=self._get_positional_embed_overrides_item(i),
# If `__getitem__` is called, these bootstrap fields must be lists.
bootstrap_host=(
+19 -2
View File
@@ -189,6 +189,11 @@ def _get_dflash_layer_attention_params(
)
_ROPE_DIM_SUPERSEDED_KEYS = frozenset(
{"rope_dim", "partial_rotary_factor", "mrope_section", "mrope_interleaved"}
)
class DFlashAttention(nn.Module):
def __init__(
self, config, layer_id: int, quant_config=None, prefix: str = ""
@@ -242,6 +247,18 @@ class DFlashAttention(nn.Module):
self.k_norm = RMSNorm(head_dim, eps=rms_norm_eps)
rope_theta, rope_scaling = get_rope_config(config)
rotary_dim = head_dim
partial_rotary_factor = float(getattr(config, "partial_rotary_factor", 1.0))
if rope_scaling is not None and "rope_dim" in rope_scaling:
# An explicit rope_dim is the trained rotary width (HF Qwen3 ignores the
# copied target partial/M-RoPE fields); drafts only see 1-D positions.
rotary_dim = int(rope_scaling["rope_dim"])
partial_rotary_factor = 1.0
rope_scaling = {
k: v
for k, v in rope_scaling.items()
if k not in _ROPE_DIM_SUPERSEDED_KEYS
}
rope_is_neox_style = bool(
getattr(
config, "rope_is_neox_style", getattr(config, "is_neox_style", True)
@@ -250,12 +267,12 @@ class DFlashAttention(nn.Module):
max_position_embeddings = int(getattr(config, "max_position_embeddings", 32768))
self.rotary_emb = get_rope(
head_dim,
rotary_dim=head_dim,
rotary_dim=rotary_dim,
max_position=max_position_embeddings,
base=rope_theta,
rope_scaling=rope_scaling,
is_neox_style=rope_is_neox_style,
partial_rotary_factor=float(getattr(config, "partial_rotary_factor", 1.0)),
partial_rotary_factor=partial_rotary_factor,
)
self.scaling = head_dim**-0.5
+137 -26
View File
@@ -21,10 +21,11 @@
# limitations under the License.
"""Inference-only MiniCPM-V model compatible with HuggingFace weights."""
import re
import types
from array import array
from functools import partial
from itertools import chain
from itertools import chain, groupby
from typing import (
Any,
Callable,
@@ -45,10 +46,12 @@ from torch import nn
from torch.nn.init import trunc_normal_
from transformers import PretrainedConfig
from sglang.srt.configs.qwen3_5 import Qwen3_5MoeTextConfig
from sglang.srt.layers.linear import ReplicatedLinear
from sglang.srt.layers.logits_processor import LogitsProcessor
from sglang.srt.layers.quantization.base_config import QuantizationConfig
from sglang.srt.managers.mm_utils import (
MultiModalityDataPaddingPatternMultimodalTokens,
MultiModalityDataPaddingPatternTokenPairs,
general_mm_embed_routine,
)
@@ -68,7 +71,8 @@ from sglang.srt.models.minicpmv_vit import (
)
from sglang.srt.models.qwen2 import Qwen2Config, Qwen2ForCausalLM
from sglang.srt.models.qwen3 import Qwen3Config, Qwen3ForCausalLM
from sglang.srt.models.qwen3_5 import Qwen3_5ForCausalLM
from sglang.srt.models.qwen3_5 import Qwen3_5ForCausalLM, Qwen3_5MoeForCausalLM
from sglang.srt.models.utils import WeightsMapper
from sglang.srt.utils import add_prefix, flatten_nested_list, get_device
RawImageType = Union[Image.Image, torch.Tensor]
@@ -571,10 +575,15 @@ class Resampler4_5(BaseResampler):
return x
# 4.6+ ship their own ``model_type``; 4.7 checkpoints still carry a stale
# numeric ``version``, so ``model_type`` must win.
_MODEL_TYPE_VERSIONS = {"minicpmv4_6": (4, 6), "minicpmv4_7": (4, 7)}
def get_version_by_config(config: PretrainedConfig) -> Tuple[int, ...]:
# 4.6 ships its own ``model_type`` instead of a numeric ``version``.
if getattr(config, "model_type", None) == "minicpmv4_6":
return 4, 6
model_type = getattr(config, "model_type", None)
if model_type in _MODEL_TYPE_VERSIONS:
return _MODEL_TYPE_VERSIONS[model_type]
version_float = getattr(config, "version", None)
@@ -1397,7 +1406,8 @@ class MiniCPMV4_6(MiniCPMBaseModel):
prefix: str = "",
):
super().__init__(config=config, quant_config=quant_config, prefix=prefix)
assert self.version == (4, 6)
assert self.version in ((4, 6), (4, 7))
self.capture_aux_hidden_states = False
# ``Qwen3_5ForCausalLM`` returns plain hidden states (body only, no LM
# head, no LogitsProcessor). Add them here so the downstream sampler
# sees a ``LogitsProcessorOutput``. With ``tie_word_embeddings=True``
@@ -1422,9 +1432,13 @@ class MiniCPMV4_6(MiniCPMBaseModel):
prefix: str = "",
) -> nn.Module:
# 4.6 nests the LLM config under ``text_config``.
return Qwen3_5ForCausalLM(
config=config.text_config, quant_config=quant_config, prefix=prefix
text_config = config.text_config
llm_cls = (
Qwen3_5MoeForCausalLM
if isinstance(text_config, Qwen3_5MoeTextConfig)
else Qwen3_5ForCausalLM
)
return llm_cls(config=text_config, quant_config=quant_config, prefix=prefix)
def forward(
self,
@@ -1442,10 +1456,22 @@ class MiniCPMV4_6(MiniCPMBaseModel):
forward_batch=forward_batch,
**kwargs,
)
aux_hidden_states = None
if self.capture_aux_hidden_states:
hidden_states, aux_hidden_states = hidden_states
return self.logits_processor(
input_ids, hidden_states, self.lm_head, forward_batch
input_ids, hidden_states, self.lm_head, forward_batch, aux_hidden_states
)
def set_dflash_layers_to_capture(self, layer_ids: List[int]):
if layer_ids is None:
raise ValueError(
"DFLASH requires explicit layer_ids for aux hidden capture."
)
self.capture_aux_hidden_states = True
# Qwen3.5 layers capture their input, i.e. the previous layer's output.
self.llm.set_dflash_layers_to_capture([val + 1 for val in layer_ids])
def init_vision_module(
self,
config: PretrainedConfig,
@@ -1500,6 +1526,22 @@ class MiniCPMV4_6(MiniCPMBaseModel):
result = torch.cat([item.feature for item in items])
return result.reshape(-1, result.shape[-1])
# ``use_vit_merger`` is per request, and one batch can mix requests.
default = self.config.downsample_mode != "4x"
runs = groupby(
items,
key=lambda item: item.model_specific_data.get("use_vit_merger", default),
)
return torch.cat(
[
self._encode_patches(list(run), use_vit_merger)
for use_vit_merger, run in runs
]
)
def _encode_patches(
self, items: List[MultimodalDataItem], use_vit_merger: bool
) -> torch.Tensor:
pixel_values = flatten_nested_list([item.feature for item in items])
tgt_sizes = torch.stack(
flatten_nested_list([item.tgt_size for item in items]), dim=0
@@ -1530,8 +1572,6 @@ class MiniCPMV4_6(MiniCPMBaseModel):
patch_attn_mask.size(2), device=patch_attn_mask.device
).unsqueeze(0) < mask_shapes.unsqueeze(1)
use_vit_merger = getattr(self.config, "downsample_mode", "16x") != "4x"
vision_embedding, tgt_sizes_out = self.vpm(
all_pixel_values.type(dtype),
patch_attention_mask=patch_attn_mask,
@@ -1546,15 +1586,8 @@ class MiniCPMV4_6(MiniCPMBaseModel):
get_video_feature = get_image_feature
def pad_input_ids(self, input_ids: array, image_inputs: MultimodalInputs) -> array:
im_start_id: int = image_inputs.im_start_id
im_end_id: int = image_inputs.im_end_id
slice_start_id: int = image_inputs.slice_start_id
slice_end_id: int = image_inputs.slice_end_id
media_token_pairs = [(im_start_id, im_end_id), (slice_start_id, slice_end_id)]
pattern = MultiModalityDataPaddingPatternTokenPairs(
media_token_pairs, data_start_token_ids=[im_start_id]
)
# Every thumbnail / slice is its own item; pad each span with its own value.
pattern = MultiModalityDataPaddingPatternMultimodalTokens()
return pattern.pad_input_tokens(input_ids, image_inputs)
def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
@@ -1606,11 +1639,65 @@ class MiniCPMV4_6(MiniCPMBaseModel):
weight_loader(param, loaded_weight)
# 4.7 checkpoints keep top-level ``llm`` / ``vpm`` / ``resampler`` / ``vit_merger``
# names and store connector MLPs as ``nn.Sequential``.
_V4_7_WEIGHT_PREFIXES = (
("llm.model.", "model.language_model."),
("llm.lm_head.", "lm_head."),
("vit_merger.", "model.vision_tower.vit_merger."),
("vpm.", "model.vision_tower."),
("resampler.", "model.merger."),
)
def _v4_7_to_v4_6_weight_name(name: str) -> str:
for old, new in _V4_7_WEIGHT_PREFIXES:
if name.startswith(old):
name = new + name[len(old) :]
break
name = re.sub(r"^(model\.merger\.mlp\.\d+)\.mlp\.0\.", r"\1.linear_1.", name)
return re.sub(r"^(model\.merger\.mlp\.\d+)\.mlp\.2\.", r"\1.linear_2.", name)
class MiniCPMV4_7(MiniCPMV4_6):
def __init__(
self,
config: PretrainedConfig,
quant_config: Optional[QuantizationConfig] = None,
prefix: str = "",
):
super().__init__(config=config, quant_config=quant_config, prefix=prefix)
rope_parameters = config.text_config.rope_parameters or {}
self.is_mrope_enabled = "mrope_section" in rope_parameters
def forward(
self,
input_ids: torch.Tensor,
positions: torch.Tensor,
forward_batch: ForwardBatch,
**kwargs: Any,
) -> torch.Tensor:
if self.is_mrope_enabled:
positions = forward_batch.mrope_positions
return super().forward(
input_ids=input_ids,
positions=positions,
forward_batch=forward_batch,
**kwargs,
)
def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
return super().load_weights(
(_v4_7_to_v4_6_weight_name(name), weight) for name, weight in weights
)
_SUPPORT_VERSION = {
(2, 6): MiniCPMV2_6,
(4, 0): MiniCPMV4_0,
(4, 5): MiniCPMV4_5,
(4, 6): MiniCPMV4_6,
(4, 7): MiniCPMV4_7,
}
@@ -1632,8 +1719,8 @@ class MiniCPMV:
@staticmethod
def shared_experts_fusion_disable_reason(hf_config, quant_config):
# 4.6 nests a Qwen3.5 LLM under ``text_config``; every other version
# builds a dense LLM, for which the Qwen3.5 gate answers None.
# 4.6/4.7 nest a Qwen3.5 config (dense or MoE) under ``text_config``;
# older versions are dense, for which the Qwen3.5 gate answers None.
return Qwen3_5ForCausalLM.shared_experts_fusion_disable_reason(
getattr(hf_config, "text_config", hf_config), quant_config
)
@@ -1646,11 +1733,12 @@ class MiniCPMV:
) -> None:
super().__init__()
# 4.6 carries ``model_type == "minicpmv4_6"`` instead of a numeric
# 4.6+ carry their own ``model_type`` instead of a numeric
# ``config.version``; older versionless configs keep the legacy
# ``(2, 6)`` default.
if getattr(config, "model_type", None) == "minicpmv4_6":
version = (4, 6)
model_type = getattr(config, "model_type", None)
if model_type in _MODEL_TYPE_VERSIONS:
version = _MODEL_TYPE_VERSIONS[model_type]
elif not hasattr(config, "version"):
version = (2, 6)
else:
@@ -1748,7 +1836,30 @@ class MiniCPMV:
# ``__name__`` — resolves the canonical 4.6 architecture name through
# ``MiniCPMV``'s version-dispatch factory.
class MiniCPMV4_6ForConditionalGeneration(MiniCPMV):
# Only for the quant config: its ``ignore`` list names modules in either the
# HF or the native 4.7 layout (see _V4_7_WEIGHT_PREFIXES), while
# load_weights remaps checkpoint names itself.
hf_to_sglang_mapper = WeightsMapper(
orig_to_new_substr={"self_attn.out_proj": "self_attn.proj"},
orig_to_new_prefix={
"model.language_model.": "llm.",
"model.vision_tower.": "vpm.",
"model.merger.": "resampler.",
"llm.model.": "llm.",
"llm.lm_head": "lm_head",
"vit_merger.": "vpm.vit_merger.",
},
orig_to_new_suffix={".mlp.0": ".linear_1", ".mlp.2": ".linear_2"},
)
packed_modules_mapping = Qwen3_5ForCausalLM.packed_modules_mapping
class MiniCPMV4_7ForConditionalGeneration(MiniCPMV4_6ForConditionalGeneration):
pass
EntryClass = [MiniCPMV, MiniCPMV4_6ForConditionalGeneration]
EntryClass = [
MiniCPMV,
MiniCPMV4_6ForConditionalGeneration,
MiniCPMV4_7ForConditionalGeneration,
]
@@ -17,6 +17,7 @@ module collapses to a thin wrapper that delegates to it.
from __future__ import annotations
import hashlib
import math
from itertools import chain
from typing import Any, List, Optional, Sequence, Tuple, Union
@@ -31,6 +32,10 @@ from sglang.srt.managers.schedule_batch import (
MultimodalProcessorOutput,
)
from sglang.srt.models.minicpmv import MiniCPMV4_6ForConditionalGeneration
from sglang.srt.multimodal.cache import (
CONTENT_HASH_PREFIX,
resolve_multimodal_item_hash,
)
from sglang.srt.multimodal.processors.base_processor import (
BaseMultimodalProcessor,
MultimodalSpecialTokens,
@@ -127,16 +132,16 @@ def _reshape_by_patch(image: torch.Tensor, patch_size: int) -> torch.Tensor:
return patches
def _flatten_patches(
per_item_pv: List[List[torch.Tensor]],
per_item_ts: List[List[List[int]]],
) -> Tuple[List[torch.Tensor], List[torch.Tensor]]:
"""Per-item per-patch -> flat per-patch (source first, slices row-major)."""
flat_pv = list(chain.from_iterable(per_item_pv))
flat_ts = [
torch.tensor(ts, dtype=torch.int32) for ts in chain.from_iterable(per_item_ts)
]
return flat_pv, flat_ts
def _layout_namespace(
downsample_mode: str,
tgt_size: List[int],
media_tgt_sizes: List[List[int]],
media_grids: List[List[int]],
) -> str:
# Pixels alone omit the layout: patches of different shape can pack to the
# same tensor while their embeddings and M-RoPE positions differ.
layout = f"{downsample_mode}|{tgt_size}|{media_tgt_sizes}|{media_grids}"
return CONTENT_HASH_PREFIX + hashlib.sha256(layout.encode()).hexdigest()
class MiniCPMV4_6ImageProcessor:
@@ -231,6 +236,14 @@ class MiniCPMV4_6ImageProcessor:
min_error = error
return best_grid
@property
def use_vit_merger(self) -> bool:
return self.downsample_mode != "4x"
@property
def pad_divisor(self) -> int:
return 16 if self.use_vit_merger else 4
def _normalize(self, t: torch.Tensor) -> torch.Tensor:
t = t * self.rescale_factor
return (t - self.image_mean.to(t.dtype)) / self.image_std.to(t.dtype)
@@ -305,6 +318,7 @@ class MiniCPMV4_6MultimodalProcessor(BaseMultimodalProcessor):
models = [MiniCPMV4_6ForConditionalGeneration]
support_dynamic_frame_expansion = False
gpu_image_decode = False
video_image_id = True
def __init__(self, hf_config, server_args, _processor, *args, **kwargs):
super().__init__(hf_config, server_args, _processor, *args, **kwargs)
@@ -353,8 +367,6 @@ class MiniCPMV4_6MultimodalProcessor(BaseMultimodalProcessor):
self.slice_start_id = self._token_id(self.slice_start_token)
self.slice_end_id = self._token_id(self.slice_end_token)
self.pad_divisor = 16 if downsample_mode != "4x" else 4
self.mm_tokens = MultimodalSpecialTokens(
image_token=self.image_token,
image_token_id=self.image_token_id,
@@ -375,19 +387,20 @@ class MiniCPMV4_6MultimodalProcessor(BaseMultimodalProcessor):
self,
tgt_sizes: List[List[int]],
grid: List[int],
pad_divisor: int,
) -> str:
"""``<image>...</image>`` (+ optional ``<slice>...</slice>`` rows) for
one image or video frame; inner pads are ``_PAD_PLACEHOLDER`` (caller
swaps back after splicing).
"""
h0, w0 = tgt_sizes[0]
n_src = (h0 * w0) // self.pad_divisor
n_src = (h0 * w0) // pad_divisor
out = self.image_start_token + _PAD_PLACEHOLDER * n_src + self.image_end_token
if len(tgt_sizes) > 1 and grid and grid[0] > 0 and grid[1] > 0:
grid_y, grid_x = int(grid[0]), int(grid[1])
h_s, w_s = tgt_sizes[1]
n_slice = (h_s * w_s) // self.pad_divisor
n_slice = (h_s * w_s) // pad_divisor
slice_chunk = (
self.slice_start_token
+ _PAD_PLACEHOLDER * n_slice
@@ -401,9 +414,13 @@ class MiniCPMV4_6MultimodalProcessor(BaseMultimodalProcessor):
self,
index: int,
frames: Sequence[Tuple[List[List[int]], List[int]]],
pad_divisor: int,
image_id: bool = True,
) -> str:
"""One image or one video. Image is a single-frame video."""
body = "".join(self._expand_frame(ts, grid) for ts, grid in frames)
body = "".join(self._expand_frame(ts, grid, pad_divisor) for ts, grid in frames)
if not image_id:
return body
return f"{self.image_id_start_token}{index}{self.image_id_end_token}" + body
async def process_mm_data_async(
@@ -433,19 +450,28 @@ class MiniCPMV4_6MultimodalProcessor(BaseMultimodalProcessor):
images = base.images or []
videos = base.videos or []
image_processor = self._image_processor_for(request_obj)
pad_divisor = image_processor.pad_divisor
# Image: one "frame" per image. Video: per-frame nesting kept so each
# frame becomes its own ``<image>...</image>`` block in the expansion.
img_per_pv, img_per_ts, img_grids = self._preprocess_images(images)
vid_per_pv, vid_per_ts, vid_grids = self._preprocess_videos(videos)
img_per_pv, img_per_ts, img_grids = self._preprocess_images(
images, image_processor
)
vid_per_pv, vid_per_ts, vid_grids = self._preprocess_videos(
videos, image_processor
)
prompt = self._splice_expansions(
prompt,
(
self._expand_media(i, [(ts, gd)])
self._expand_media(i, [(ts, gd)], pad_divisor)
for i, (ts, gd) in enumerate(zip(img_per_ts, img_grids))
),
(
self._expand_media(i, list(zip(fts, fgd)))
self._expand_media(
i, list(zip(fts, fgd)), pad_divisor, self.video_image_id
)
for i, (fts, fgd) in enumerate(zip(vid_per_ts, vid_grids))
),
)
@@ -462,20 +488,29 @@ class MiniCPMV4_6MultimodalProcessor(BaseMultimodalProcessor):
self._build_items(
input_ids_tensor,
self.image_token_id,
_flatten_patches(img_per_pv, img_per_ts),
[
(pvs, tss, [grid])
for pvs, tss, grid in zip(img_per_pv, img_per_ts, img_grids)
],
Modality.IMAGE,
image_processor,
)
)
# Video: extra ``per-frame -> per-patch`` nesting; pre-flatten one
# level so ``_flatten_patches`` sees the same shape as image.
vid_pv_flat = [list(chain.from_iterable(v)) for v in vid_per_pv]
vid_ts_flat = [list(chain.from_iterable(v)) for v in vid_per_ts]
# Video: one media unit per video, its frames' patches flattened in order.
mm_items.extend(
self._build_items(
input_ids_tensor,
self.video_token_id,
_flatten_patches(vid_pv_flat, vid_ts_flat),
[
(
list(chain.from_iterable(pvs)),
list(chain.from_iterable(tss)),
grids,
)
for pvs, tss, grids in zip(vid_per_pv, vid_per_ts, vid_grids)
],
Modality.VIDEO,
image_processor,
)
)
@@ -483,24 +518,28 @@ class MiniCPMV4_6MultimodalProcessor(BaseMultimodalProcessor):
mm_items=mm_items,
input_ids=input_ids,
im_token_id=self.image_token_id,
video_token_id=self.video_token_id,
im_start_id=self.image_start_id,
im_end_id=self.image_end_id,
slice_start_id=self.slice_start_id,
slice_end_id=self.slice_end_id,
)
def _preprocess_images(self, images):
def _image_processor_for(self, request_obj) -> MiniCPMV4_6ImageProcessor:
return self.image_processor
def _preprocess_images(self, images, image_processor):
if not images:
return [], [], []
out = self.image_processor.preprocess(images)
out = image_processor.preprocess(images)
return out["pixel_values"], out["tgt_sizes"], out["grids"]
def _preprocess_videos(self, videos):
def _preprocess_videos(self, videos, image_processor):
per_video_pv: List[List[List[torch.Tensor]]] = []
per_video_ts: List[List[List[List[int]]]] = []
per_video_grids: List[List[List[int]]] = []
for frames in videos:
out = self.image_processor.preprocess(list(frames))
out = image_processor.preprocess(list(frames))
per_video_pv.append(out["pixel_values"])
per_video_ts.append(out["tgt_sizes"])
per_video_grids.append(out["grids"])
@@ -527,22 +566,33 @@ class MiniCPMV4_6MultimodalProcessor(BaseMultimodalProcessor):
self,
input_ids: torch.Tensor,
pad_token_id: int,
flat: Tuple[List[torch.Tensor], List[torch.Tensor]],
media: List[Tuple[List[torch.Tensor], List[List[int]], List[List[int]]]],
modality: Modality,
image_processor: MiniCPMV4_6ImageProcessor,
) -> List[MultimodalDataItem]:
flat_pv, flat_ts = flat
"""``media``: per image / video, its patches' pixel values, tgt sizes
and slice grids (source patch first, slices row-major)."""
patches = [
(pv, ts, _layout_namespace(image_processor.downsample_mode, ts, tss, grids))
for pvs, tss, grids in media
for pv, ts in zip(pvs, tss)
]
runs = self.get_mm_items_offset(input_ids, pad_token_id)
if len(runs) != len(flat_pv):
if len(runs) != len(patches):
raise RuntimeError(
f"[minicpmv4_6] {modality} pad run / feature count mismatch: "
f"{len(runs)} runs vs {len(flat_pv)} patches"
f"{len(runs)} runs vs {len(patches)} patches"
)
return [
MultimodalDataItem(
feature=[pv],
hash=resolve_multimodal_item_hash(feature=[pv], namespace=namespace),
offsets=[run],
model_specific_data={"tgt_size": [ts]},
model_specific_data={
"tgt_size": [torch.tensor(ts, dtype=torch.int32)],
"use_vit_merger": image_processor.use_vit_merger,
},
modality=modality,
)
for run, pv, ts in zip(runs, flat_pv, flat_ts)
for run, (pv, ts, namespace) in zip(runs, patches)
]
@@ -0,0 +1,347 @@
# Copyright 2026 The SGLang team.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
"""SGLang multimodal processor for MiniCPM-V 4.7.
The canvas M-RoPE layout ports the checkpoints' reference
``mrope_minicpmv4_7.py`` and must assign exactly the same positions.
"""
from __future__ import annotations
import logging
import math
from typing import List, Optional, Tuple
import torch
from sglang.srt.models.minicpmv import MiniCPMV4_7ForConditionalGeneration
from sglang.srt.multimodal.processors.minicpmv4_6 import (
MiniCPMV4_6ImageProcessor,
MiniCPMV4_6MultimodalProcessor,
)
logger = logging.getLogger(__name__)
_SCOPED_KWARGS_KEYS = ("images_kwargs", "videos_kwargs", "audio_kwargs")
_STRUCTURAL_KEYS = ("im_start_id", "im_end_id", "slice_start_id", "slice_end_id")
# (start, end, index into target_sizes) of one run of visual tokens.
_Span = Tuple[int, int, int]
_Frame = Tuple[_Span, List[_Span]]
def _parse_mm_processor_kwargs(raw) -> dict:
"""Flatten HF-style ``images_kwargs`` / ``videos_kwargs``; top-level keys win."""
if not isinstance(raw, dict):
return {}
flat = {}
for key in _SCOPED_KWARGS_KEYS:
if isinstance(raw.get(key), dict):
flat.update(raw[key])
flat.update({k: v for k, v in raw.items() if k not in _SCOPED_KWARGS_KEYS})
return flat
def _llm_grid(n_tokens: int, vit_h: int, vit_w: int) -> Tuple[int, int]:
"""LLM-token grid for ``n_tokens`` merged from a ``vit_h x vit_w`` patch grid."""
if n_tokens <= 0 or vit_h <= 0 or vit_w <= 0:
return 0, 0
vit_total = vit_h * vit_w
if vit_total == n_tokens:
return vit_h, vit_w
if vit_total < n_tokens:
return 0, 0
factor = int(round(math.sqrt(vit_total / n_tokens)))
if factor > 0 and vit_h % factor == 0 and vit_w % factor == 0:
height, width = vit_h // factor, vit_w // factor
if height * width == n_tokens:
return height, width
ratio = vit_h / vit_w
width = max(1, int(round(math.sqrt(n_tokens / ratio))))
height = n_tokens // width
if height * width == n_tokens and height > 0:
return height, width
return 0, 0
def build_image_bounds(input_ids: torch.Tensor, token_ids: dict) -> torch.Tensor:
"""``[start, end)`` of every thumbnail / slice token run, shape (n, 2)."""
start_ids = [token_ids[k] for k in ("im_start_id", "slice_start_id")]
end_ids = [token_ids[k] for k in ("im_end_id", "slice_end_id")]
starts = torch.isin(input_ids, torch.tensor(start_ids)).nonzero().flatten() + 1
ends = torch.isin(input_ids, torch.tensor(end_ids)).nonzero().flatten()
n = min(len(starts), len(ends))
return torch.stack([starts[:n], ends[:n]], dim=-1)
def _frame_end(ids: List[int], frame: _Frame, structural: set, limit: int) -> int:
thumbnail, slices = frame
end = slices[-1][1] if slices else thumbnail[1]
while end < limit and ids[end] in structural:
end += 1
return min(end, limit)
def _group_frames(
ids: List[int], bounds: List[List[int]], token_ids: dict, structural: set
) -> List[List[_Frame]]:
"""Pair each thumbnail with its slices, then merge frames separated only by
structural / newline tokens into one video group."""
frames: List[_Frame] = []
for index, (start, end) in enumerate(bounds):
if frames and ids[start - 1] == token_ids["slice_start_id"]:
frames[-1][1].append((start, end, index))
else:
frames.append(((start, end, index), []))
gap_ok = structural | {token_ids["newline_id"]}
groups = [[frames[0]]]
for prev, frame in zip(frames, frames[1:]):
gap = range(_frame_end(ids, prev, structural, len(ids)), frame[0][0] - 1)
if all(ids[p] in gap_ok for p in gap):
groups[-1].append(frame)
else:
groups.append([frame])
return groups
def _fill_text(positions: torch.Tensor, start: int, end: int, pos: int) -> int:
if end <= start:
return pos
positions[:, start:end] = torch.arange(pos, pos + end - start)
return pos + end - start
def _fill_grid(
positions: torch.Tensor,
start: int,
base: int,
rows: torch.Tensor,
cols: torch.Tensor,
) -> None:
end = start + len(rows) * len(cols)
positions[0, start:end] = base
positions[1, start:end] = rows.repeat_interleave(len(cols)) + base
positions[2, start:end] = cols.repeat(len(rows)) + base
def _place_frame(
positions: torch.Tensor,
frame: _Frame,
frame_end: int,
base: int,
sizes: List[List[int]],
) -> int:
"""Lay one thumbnail and its slice grid out on a shared canvas at ``base``;
returns the first position after the canvas."""
(t_start, t_end, t_index), slices = frame
im_start = t_start - 1
n_thumb = t_end - t_start
th, tw = _llm_grid(n_thumb, *sizes[t_index])
if slices:
cols = len(slices)
for k in range(len(slices) - 1):
if slices[k + 1][0] - slices[k][1] > 2:
cols = k + 1
break
rows = len(slices) // cols
if rows * cols != len(slices):
rows, cols = 1, len(slices)
s0_start, s0_end, s0_index = slices[0]
sh, sw = _llm_grid(s0_end - s0_start, *sizes[s0_index])
canvas_h, canvas_w = rows * sh, cols * sw
else:
canvas_h, canvas_w = max(th, 1), max(tw, 1)
positions[:, im_start:frame_end] = base
positions[1:, im_start] = max(base - 1, 0)
if t_end < frame_end:
positions[1, t_end] = base + canvas_h
positions[2, t_end] = base + canvas_w
if th > 0:
if slices and canvas_h > 0 and canvas_w > 0:
# The thumbnail is stretched over the full slice canvas.
t_rows = torch.linspace(0, canvas_h - 1, th).round().long()
t_cols = torch.linspace(0, canvas_w - 1, tw).round().long()
else:
t_rows, t_cols = torch.arange(th), torch.arange(tw)
_fill_grid(positions, t_start, base, t_rows, t_cols)
else:
_fill_text(positions, t_start, t_end, base)
for k, (s_start, s_end, s_index) in enumerate(slices):
sh_k, sw_k = _llm_grid(s_end - s_start, *sizes[s_index])
h_off, w_off = (k // cols) * sh, (k % cols) * sw
positions[1, s_start - 1] = base + h_off
positions[2, s_start - 1] = base + w_off
if s_end < frame_end:
positions[1, s_end] = base + h_off + max(sh_k - 1, 0)
positions[2, s_end] = base + w_off + max(sw_k - 1, 0)
if sh_k > 0:
rows_k = torch.arange(sh_k) + h_off
cols_k = torch.arange(sw_k) + w_off
_fill_grid(positions, s_start, base, rows_k, cols_k)
else:
_fill_text(positions, s_start, s_end, base)
# Row-separating newlines sit one column past the canvas, on the row just ended.
for k in range(len(slices) - 1):
gap_start, gap_end = slices[k][1] + 1, slices[k + 1][0] - 1
if gap_end > gap_start:
positions[1, gap_start:gap_end] = base + (k // cols + 1) * sh - 1
positions[2, gap_start:gap_end] = base + cols * sw
return base + max(canvas_h, canvas_w) + 1
def compute_canvas_positions(
input_ids: torch.Tensor,
image_bounds: torch.Tensor,
target_sizes: List[List[int]],
token_ids: dict,
) -> torch.Tensor:
"""(3, seq_len) ``[temporal, height, width]`` canvas M-RoPE positions."""
ids = input_ids.tolist()
seq_len = len(ids)
positions = torch.arange(seq_len).expand(3, -1).clone()
if len(image_bounds) == 0:
return positions
structural = {token_ids[k] for k in _STRUCTURAL_KEYS}
pos = cursor = 0
for frames in _group_frames(ids, image_bounds.tolist(), token_ids, structural):
group_start = frames[0][0][0] - 1
group_end = _frame_end(ids, frames[-1], structural, seq_len)
pos = _fill_text(positions, cursor, group_start, pos)
frame_cursor = group_start
for frame in frames:
im_start = frame[0][0] - 1
if im_start > frame_cursor:
# The extra step keeps the next frame's <image> halo off the gap.
pos = _fill_text(positions, frame_cursor, im_start, pos) + 1
frame_end = _frame_end(ids, frame, structural, group_end)
pos = _place_frame(positions, frame, frame_end, pos, target_sizes)
frame_cursor = frame_end
cursor = group_end
_fill_text(positions, cursor, seq_len, pos)
return positions
def _sequential_positions(seq_len: int) -> torch.Tensor:
return torch.arange(seq_len).expand(3, -1)
def _rope_delta(positions: torch.Tensor) -> torch.Tensor:
"""Decode offset: the first generated token sits one past the last prompt position."""
delta = int(positions.amax()) + 1 - positions.shape[1]
return torch.tensor([[delta]], dtype=torch.int64)
class MiniCPMV4_7MultimodalProcessor(MiniCPMV4_6MultimodalProcessor):
models = [MiniCPMV4_7ForConditionalGeneration]
# 4.7 was trained without ``<image_id>`` in front of videos.
video_image_id = False
def __init__(self, hf_config, server_args, _processor, *args, **kwargs):
super().__init__(hf_config, server_args, _processor, *args, **kwargs)
self.canvas_token_ids = self._resolve_canvas_token_ids(hf_config)
def _image_processor_for(self, request_obj) -> MiniCPMV4_6ImageProcessor:
kwargs = _parse_mm_processor_kwargs(
getattr(request_obj, "mm_processor_kwargs", None)
)
default = self.image_processor
downsample_mode = kwargs.get("downsample_mode") or default.downsample_mode
if downsample_mode not in ("4x", "16x"):
logger.warning("Ignoring unknown downsample_mode %r", downsample_mode)
downsample_mode = default.downsample_mode
max_slice_nums = kwargs.get("max_slice_nums")
if max_slice_nums is None:
max_slice_nums = default.max_slice_nums
try:
max_slice_nums = int(max_slice_nums)
except (TypeError, ValueError):
logger.warning("Ignoring non-numeric max_slice_nums %r", max_slice_nums)
max_slice_nums = default.max_slice_nums
if (downsample_mode, max_slice_nums) == (
default.downsample_mode,
default.max_slice_nums,
):
return default
return MiniCPMV4_6ImageProcessor(
max_slice_nums=max_slice_nums,
scale_resolution=default.scale_resolution,
patch_size=default.patch_size,
downsample_mode=downsample_mode,
)
def _resolve_canvas_token_ids(self, hf_config) -> Optional[dict]:
mrope_mode = str(getattr(hf_config, "mrope_mode", "")).strip().lower()
if (
not getattr(hf_config, "uses_mrope_canvas", False)
and mrope_mode != "canvas"
):
return None
# Byte-level BPE vocabularies store "\n" as "\u010a", so encode it.
newline_ids = self.tokenizer.encode("\n", add_special_tokens=False)
from_tokenizer = {
"im_start_id": self.image_start_id,
"im_end_id": self.image_end_id,
"slice_start_id": self.slice_start_id,
"slice_end_id": self.slice_end_id,
"newline_id": newline_ids[0] if len(newline_ids) == 1 else None,
}
token_ids = {
"im_start_id": getattr(hf_config, "image_start_id", None),
"im_end_id": getattr(hf_config, "image_end_id", None),
"slice_start_id": getattr(hf_config, "slice_start_id", None),
"slice_end_id": getattr(hf_config, "slice_end_id", None),
"newline_id": getattr(hf_config, "newline_id", None),
}
token_ids = {
k: from_tokenizer[k] if v is None else v for k, v in token_ids.items()
}
if None in token_ids.values():
logger.warning("Canvas M-RoPE disabled: token ids missing (%s)", token_ids)
return None
return {k: int(v) for k, v in token_ids.items()}
def compute_mrope_positions(self, input_ids, mm_items):
ids = torch.tensor(list(input_ids), dtype=torch.long)
positions = self._canvas_positions(ids, mm_items)
# The Qwen3.5 backbone reads mrope_positions for every multimodal request.
if positions is None:
positions = _sequential_positions(len(ids))
return positions, _rope_delta(positions)
def _canvas_positions(self, ids, mm_items) -> Optional[torch.Tensor]:
if self.canvas_token_ids is None:
return None
# Items are appended images-first; the canvas walk needs them in prompt order.
ordered = sorted(mm_items, key=lambda item: item.offsets[0][0])
target_sizes = [torch.as_tensor(item.tgt_size[0]).tolist() for item in ordered]
image_bounds = build_image_bounds(ids, self.canvas_token_ids)
if len(image_bounds) != len(target_sizes):
logger.warning(
"Canvas M-RoPE: %d image bounds vs %d patch grids; "
"using sequential positions",
len(image_bounds),
len(target_sizes),
)
return None
return compute_canvas_positions(
ids, image_bounds, target_sizes, self.canvas_token_ids
)
__all__ = ["MiniCPMV4_7MultimodalProcessor"]
+2 -2
View File
@@ -1196,11 +1196,11 @@ def match_minicpm(model_path: str):
# must NOT fall back to the legacy `minicpmv` conv template (which encodes
# the old `(<image>./</image>)` placeholder used by 2.x/4.0/4.5).
model_type = get_model_type(model_path)
if model_type == "minicpmv4_6":
if model_type in ("minicpmv4_6", "minicpmv4_7"):
return None
# For HF-hub paths (where config.json isn't on local disk yet), fall back
# to a path-version check: exclude 4.6 and later, match only legacy 2.x/4.0/4.5.
if re.search(r"minicpm-(v|o)-4[._]6", model_path, re.IGNORECASE):
if re.search(r"minicpm-(v|o)-4[._][67]", model_path, re.IGNORECASE):
return None
match = re.search(r"minicpm-(v|o)", model_path, re.IGNORECASE)
if match:
@@ -193,8 +193,8 @@ def _get_text_config(config: Any) -> Any:
return config
def _get_dspark_config(config: Any) -> dict:
cfg = _cfg_get(config, "dspark_config", None)
def _get_nested_config(config: Any, key: str) -> dict:
cfg = _cfg_get(config, key, None)
if cfg is None:
return {}
if isinstance(cfg, dict):
@@ -205,6 +205,14 @@ def _get_dspark_config(config: Any) -> dict:
return {}
def _get_dspark_config(config: Any) -> dict:
# Published DSpark drafts nest their fields under dflash_config.
return {
**_get_nested_config(config, "dflash_config"),
**_get_nested_config(config, "dspark_config"),
}
def parse_dspark_draft_config(*, draft_hf_config: Any) -> DSparkDraftConfig:
base = parse_dflash_draft_config(draft_hf_config=draft_hf_config)
@@ -69,6 +69,7 @@ from sglang.srt.configs import (
MiniCPMHybridConfig,
MiniCPMV4_6Config,
MiniCPMV4_6VisionConfig,
MiniCPMV4_7Config,
MiniMaxM3VLConfig,
MultiModalityConfig,
MuseGlimmerAssistantConfig,
@@ -176,6 +177,7 @@ _CONFIG_REGISTRY: Dict[str, Type[PretrainedConfig]] = {
MiniCPMHybridConfig,
MiniCPMV4_6Config,
MiniCPMV4_6VisionConfig,
MiniCPMV4_7Config,
InklingModelConfig,
InklingAudioConfig,
InklingVisionConfig,