diff --git a/python/sglang/srt/arg_groups/model_overrides/minicpmv.py b/python/sglang/srt/arg_groups/model_overrides/minicpmv.py
index 9fe11f0b10..a28de2e0a5 100644
--- a/python/sglang/srt/arg_groups/model_overrides/minicpmv.py
+++ b/python/sglang/srt/arg_groups/model_overrides/minicpmv.py
@@ -10,6 +10,7 @@ from sglang.srt.runtime_context import get_platform
@_register_for("MiniCPMV4_6ForConditionalGeneration")
+@_register_for("MiniCPMV4_7ForConditionalGeneration")
def _minicpm_v4_6_overrides(server_args: Any, hf_config: Any) -> dict:
cfg = resolving_view(server_args)
if get_platform().is_sm100 and cfg.attention_backend is None:
diff --git a/python/sglang/srt/arg_groups/overrides.py b/python/sglang/srt/arg_groups/overrides.py
index ac79251c93..b30360d623 100644
--- a/python/sglang/srt/arg_groups/overrides.py
+++ b/python/sglang/srt/arg_groups/overrides.py
@@ -362,6 +362,7 @@ _MAMBA_RADIX_CACHE_ARCHS = frozenset(
"Qwen3_5ForCausalLM",
"Qwen4ExpForConditionalGeneration",
"MiniCPMV4_6ForConditionalGeneration",
+ "MiniCPMV4_7ForConditionalGeneration",
"NemotronHForCausalLM",
"NemotronHPuzzleForCausalLM",
"NemotronH_Omni_Reasoning_V3",
@@ -390,6 +391,7 @@ _MAMBA_EXTRA_BUFFER_ARCHS = frozenset(
"Qwen4ExpForConditionalGeneration",
"InternS2PreviewForConditionalGeneration",
"MiniCPMV4_6ForConditionalGeneration",
+ "MiniCPMV4_7ForConditionalGeneration",
"BailingMoeV2_5ForCausalLM",
"BailingMoeV3ForCausalLM",
"BailingMoeV3VLForConditionalGeneration",
diff --git a/python/sglang/srt/configs/__init__.py b/python/sglang/srt/configs/__init__.py
index 06239f17ee..69f21c1ae6 100644
--- a/python/sglang/srt/configs/__init__.py
+++ b/python/sglang/srt/configs/__init__.py
@@ -56,6 +56,7 @@ from sglang.srt.configs.mamba import FalconMambaConfig, MambaConfig
from sglang.srt.configs.mamba2 import Mamba2Config
from sglang.srt.configs.minicpm import MiniCPMHybridConfig
from sglang.srt.configs.minicpmv4_6 import MiniCPMV4_6Config, MiniCPMV4_6VisionConfig
+from sglang.srt.configs.minicpmv4_7 import MiniCPMV4_7Config
from sglang.srt.configs.minimax_vl import MiniMaxM3VLConfig
from sglang.srt.configs.muse_glimmer import (
MuseGlimmerAssistantConfig,
@@ -150,6 +151,7 @@ __all__ = [
"LocateAnythingConfig",
"MiniCPMV4_6Config",
"MiniCPMV4_6VisionConfig",
+ "MiniCPMV4_7Config",
"NemotronHConfig",
"NemotronHPuzzleConfig",
"NemotronH_Nano_VL_V2_Config",
diff --git a/python/sglang/srt/configs/minicpmv4_6.py b/python/sglang/srt/configs/minicpmv4_6.py
index 472224a892..69dae04518 100644
--- a/python/sglang/srt/configs/minicpmv4_6.py
+++ b/python/sglang/srt/configs/minicpmv4_6.py
@@ -17,7 +17,7 @@ from typing import Any, Dict, Optional, Union
from transformers import AutoConfig, PretrainedConfig
from transformers.models.auto import CONFIG_MAPPING
-from sglang.srt.configs.qwen3_5 import Qwen3_5TextConfig
+from sglang.srt.configs.qwen3_5 import Qwen3_5MoeTextConfig, Qwen3_5TextConfig
class MiniCPMV4_6VisionConfig(PretrainedConfig):
@@ -54,15 +54,17 @@ class MiniCPMV4_6VisionConfig(PretrainedConfig):
def _resolve_text_config_class(model_type: Optional[str]) -> type:
- """``model_type`` -> registered config class. sglang's ``Qwen3_5TextConfig``
- wins over the stock entry when both exist (it carries ``layers_block_type``
- etc. that the model code reads); ``AutoConfig.register`` doesn't replace
- existing entries so we have to short-circuit here. Note that
+ """``model_type`` -> registered config class. sglang's Qwen3.5 dense / MoE
+ text configs win over the stock entries when both exist (they carry
+ ``layers_block_type`` etc. that the model code reads);
+ ``AutoConfig.register`` doesn't replace existing entries so we have to
+ short-circuit here. Note that
``CONFIG_MAPPING.get`` returns ``None`` even on hit — go through
``__getitem__`` to trigger the lazy class import.
"""
- if model_type == Qwen3_5TextConfig.model_type:
- return Qwen3_5TextConfig
+ for cls in (Qwen3_5TextConfig, Qwen3_5MoeTextConfig):
+ if model_type == cls.model_type:
+ return cls
if model_type and model_type in CONFIG_MAPPING:
return CONFIG_MAPPING[model_type]
raise KeyError(f"Unknown text_config model_type: {model_type!r}")
diff --git a/python/sglang/srt/configs/minicpmv4_7.py b/python/sglang/srt/configs/minicpmv4_7.py
new file mode 100644
index 0000000000..2a52a73412
--- /dev/null
+++ b/python/sglang/srt/configs/minicpmv4_7.py
@@ -0,0 +1,22 @@
+# Copyright 2026 The SGLang team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+from typing import Any
+
+from sglang.srt.configs.minicpmv4_6 import MiniCPMV4_6Config
+
+
+class MiniCPMV4_7Config(MiniCPMV4_6Config):
+ model_type = "minicpmv4_7"
+
+ # transformers 5 wraps config subclasses in ``@dataclass(kw_only=True)``;
+ # its generated ``__init__`` would skip ``MiniCPMV4_6Config.__init__``.
+ def __init__(self, **kwargs: Any) -> None:
+ super().__init__(**kwargs)
+
+
+__all__ = ["MiniCPMV4_7Config"]
diff --git a/python/sglang/srt/entrypoints/openai/protocol.py b/python/sglang/srt/entrypoints/openai/protocol.py
index 3bb5af6a06..36c1a41f67 100644
--- a/python/sglang/srt/entrypoints/openai/protocol.py
+++ b/python/sglang/srt/entrypoints/openai/protocol.py
@@ -407,6 +407,9 @@ class CompletionRequest(PDRoutingFields):
)
images_config: Optional[Dict] = None
+ # Per-request processor overrides; only MiniCPM-V 4.7 reads them (downsample_mode,
+ # max_slice_nums), flattening images_/videos_kwargs into one request-wide profile.
+ mm_processor_kwargs: Optional[Dict] = None
# Deprecated: use routed_dp_rank instead
data_parallel_rank: Optional[int] = None
@@ -966,6 +969,9 @@ class ChatCompletionRequest(PDRoutingFields):
images_config: Optional[Dict] = None
video_config: Optional[Dict] = None
+ # Per-request processor overrides; only MiniCPM-V 4.7 reads them (downsample_mode,
+ # max_slice_nums), flattening images_/videos_kwargs into one request-wide profile.
+ mm_processor_kwargs: Optional[Dict] = None
# Custom logit processor for advanced sampling control
custom_logit_processor: Optional[Union[List[Optional[str]], str]] = None
diff --git a/python/sglang/srt/entrypoints/openai/serving_chat.py b/python/sglang/srt/entrypoints/openai/serving_chat.py
index 4b361f28f7..ec56b6f709 100644
--- a/python/sglang/srt/entrypoints/openai/serving_chat.py
+++ b/python/sglang/srt/entrypoints/openai/serving_chat.py
@@ -1328,6 +1328,7 @@ class OpenAIServingChat(OpenAIServingBase):
custom_logit_processor=request.custom_logit_processor,
images_config=getattr(request, "images_config", None),
video_config=_build_video_config(request),
+ mm_processor_kwargs=request.mm_processor_kwargs,
image_max_dynamic_patch=img_max_dynamic_patch,
video_max_dynamic_patch=vid_max_dynamic_patch,
max_dynamic_patch=getattr(request, "max_dynamic_patch", None),
diff --git a/python/sglang/srt/entrypoints/openai/serving_completions.py b/python/sglang/srt/entrypoints/openai/serving_completions.py
index c2e903e8e5..e381ddc396 100644
--- a/python/sglang/srt/entrypoints/openai/serving_completions.py
+++ b/python/sglang/srt/entrypoints/openai/serving_completions.py
@@ -140,6 +140,7 @@ class OpenAIServingCompletion(OpenAIServingBase):
custom_labels=custom_labels,
custom_logit_processor=request.custom_logit_processor,
images_config=getattr(request, "images_config", None),
+ mm_processor_kwargs=request.mm_processor_kwargs,
)
return adapted_request, request
diff --git a/python/sglang/srt/managers/io_struct.py b/python/sglang/srt/managers/io_struct.py
index 45d6228dab..e645e0230f 100644
--- a/python/sglang/srt/managers/io_struct.py
+++ b/python/sglang/srt/managers/io_struct.py
@@ -355,6 +355,7 @@ class GenerateReqInput:
# For Unlimited-OCR
images_config: Optional[dict] = None
+ mm_processor_kwargs: Optional[dict] = None
# Pre-computed delimiter indices for multi-item scoring.
# Batch-level: List[List[int]] (one per request). After __getitem__: List[int].
@@ -1009,6 +1010,7 @@ class GenerateReqInput:
if self.custom_logit_processor is not None
else None
),
+ mm_processor_kwargs=self.mm_processor_kwargs,
positional_embed_overrides=self._get_positional_embed_overrides_item(i),
# If `__getitem__` is called, these bootstrap fields must be lists.
bootstrap_host=(
diff --git a/python/sglang/srt/models/dflash.py b/python/sglang/srt/models/dflash.py
index 62f823be4f..03b3837779 100644
--- a/python/sglang/srt/models/dflash.py
+++ b/python/sglang/srt/models/dflash.py
@@ -189,6 +189,11 @@ def _get_dflash_layer_attention_params(
)
+_ROPE_DIM_SUPERSEDED_KEYS = frozenset(
+ {"rope_dim", "partial_rotary_factor", "mrope_section", "mrope_interleaved"}
+)
+
+
class DFlashAttention(nn.Module):
def __init__(
self, config, layer_id: int, quant_config=None, prefix: str = ""
@@ -242,6 +247,18 @@ class DFlashAttention(nn.Module):
self.k_norm = RMSNorm(head_dim, eps=rms_norm_eps)
rope_theta, rope_scaling = get_rope_config(config)
+ rotary_dim = head_dim
+ partial_rotary_factor = float(getattr(config, "partial_rotary_factor", 1.0))
+ if rope_scaling is not None and "rope_dim" in rope_scaling:
+ # An explicit rope_dim is the trained rotary width (HF Qwen3 ignores the
+ # copied target partial/M-RoPE fields); drafts only see 1-D positions.
+ rotary_dim = int(rope_scaling["rope_dim"])
+ partial_rotary_factor = 1.0
+ rope_scaling = {
+ k: v
+ for k, v in rope_scaling.items()
+ if k not in _ROPE_DIM_SUPERSEDED_KEYS
+ }
rope_is_neox_style = bool(
getattr(
config, "rope_is_neox_style", getattr(config, "is_neox_style", True)
@@ -250,12 +267,12 @@ class DFlashAttention(nn.Module):
max_position_embeddings = int(getattr(config, "max_position_embeddings", 32768))
self.rotary_emb = get_rope(
head_dim,
- rotary_dim=head_dim,
+ rotary_dim=rotary_dim,
max_position=max_position_embeddings,
base=rope_theta,
rope_scaling=rope_scaling,
is_neox_style=rope_is_neox_style,
- partial_rotary_factor=float(getattr(config, "partial_rotary_factor", 1.0)),
+ partial_rotary_factor=partial_rotary_factor,
)
self.scaling = head_dim**-0.5
diff --git a/python/sglang/srt/models/minicpmv.py b/python/sglang/srt/models/minicpmv.py
index 4fa7b66a6e..87fa8bbaa5 100644
--- a/python/sglang/srt/models/minicpmv.py
+++ b/python/sglang/srt/models/minicpmv.py
@@ -21,10 +21,11 @@
# limitations under the License.
"""Inference-only MiniCPM-V model compatible with HuggingFace weights."""
+import re
import types
from array import array
from functools import partial
-from itertools import chain
+from itertools import chain, groupby
from typing import (
Any,
Callable,
@@ -45,10 +46,12 @@ from torch import nn
from torch.nn.init import trunc_normal_
from transformers import PretrainedConfig
+from sglang.srt.configs.qwen3_5 import Qwen3_5MoeTextConfig
from sglang.srt.layers.linear import ReplicatedLinear
from sglang.srt.layers.logits_processor import LogitsProcessor
from sglang.srt.layers.quantization.base_config import QuantizationConfig
from sglang.srt.managers.mm_utils import (
+ MultiModalityDataPaddingPatternMultimodalTokens,
MultiModalityDataPaddingPatternTokenPairs,
general_mm_embed_routine,
)
@@ -68,7 +71,8 @@ from sglang.srt.models.minicpmv_vit import (
)
from sglang.srt.models.qwen2 import Qwen2Config, Qwen2ForCausalLM
from sglang.srt.models.qwen3 import Qwen3Config, Qwen3ForCausalLM
-from sglang.srt.models.qwen3_5 import Qwen3_5ForCausalLM
+from sglang.srt.models.qwen3_5 import Qwen3_5ForCausalLM, Qwen3_5MoeForCausalLM
+from sglang.srt.models.utils import WeightsMapper
from sglang.srt.utils import add_prefix, flatten_nested_list, get_device
RawImageType = Union[Image.Image, torch.Tensor]
@@ -571,10 +575,15 @@ class Resampler4_5(BaseResampler):
return x
+# 4.6+ ship their own ``model_type``; 4.7 checkpoints still carry a stale
+# numeric ``version``, so ``model_type`` must win.
+_MODEL_TYPE_VERSIONS = {"minicpmv4_6": (4, 6), "minicpmv4_7": (4, 7)}
+
+
def get_version_by_config(config: PretrainedConfig) -> Tuple[int, ...]:
- # 4.6 ships its own ``model_type`` instead of a numeric ``version``.
- if getattr(config, "model_type", None) == "minicpmv4_6":
- return 4, 6
+ model_type = getattr(config, "model_type", None)
+ if model_type in _MODEL_TYPE_VERSIONS:
+ return _MODEL_TYPE_VERSIONS[model_type]
version_float = getattr(config, "version", None)
@@ -1397,7 +1406,8 @@ class MiniCPMV4_6(MiniCPMBaseModel):
prefix: str = "",
):
super().__init__(config=config, quant_config=quant_config, prefix=prefix)
- assert self.version == (4, 6)
+ assert self.version in ((4, 6), (4, 7))
+ self.capture_aux_hidden_states = False
# ``Qwen3_5ForCausalLM`` returns plain hidden states (body only, no LM
# head, no LogitsProcessor). Add them here so the downstream sampler
# sees a ``LogitsProcessorOutput``. With ``tie_word_embeddings=True``
@@ -1422,9 +1432,13 @@ class MiniCPMV4_6(MiniCPMBaseModel):
prefix: str = "",
) -> nn.Module:
# 4.6 nests the LLM config under ``text_config``.
- return Qwen3_5ForCausalLM(
- config=config.text_config, quant_config=quant_config, prefix=prefix
+ text_config = config.text_config
+ llm_cls = (
+ Qwen3_5MoeForCausalLM
+ if isinstance(text_config, Qwen3_5MoeTextConfig)
+ else Qwen3_5ForCausalLM
)
+ return llm_cls(config=text_config, quant_config=quant_config, prefix=prefix)
def forward(
self,
@@ -1442,10 +1456,22 @@ class MiniCPMV4_6(MiniCPMBaseModel):
forward_batch=forward_batch,
**kwargs,
)
+ aux_hidden_states = None
+ if self.capture_aux_hidden_states:
+ hidden_states, aux_hidden_states = hidden_states
return self.logits_processor(
- input_ids, hidden_states, self.lm_head, forward_batch
+ input_ids, hidden_states, self.lm_head, forward_batch, aux_hidden_states
)
+ def set_dflash_layers_to_capture(self, layer_ids: List[int]):
+ if layer_ids is None:
+ raise ValueError(
+ "DFLASH requires explicit layer_ids for aux hidden capture."
+ )
+ self.capture_aux_hidden_states = True
+ # Qwen3.5 layers capture their input, i.e. the previous layer's output.
+ self.llm.set_dflash_layers_to_capture([val + 1 for val in layer_ids])
+
def init_vision_module(
self,
config: PretrainedConfig,
@@ -1500,6 +1526,22 @@ class MiniCPMV4_6(MiniCPMBaseModel):
result = torch.cat([item.feature for item in items])
return result.reshape(-1, result.shape[-1])
+ # ``use_vit_merger`` is per request, and one batch can mix requests.
+ default = self.config.downsample_mode != "4x"
+ runs = groupby(
+ items,
+ key=lambda item: item.model_specific_data.get("use_vit_merger", default),
+ )
+ return torch.cat(
+ [
+ self._encode_patches(list(run), use_vit_merger)
+ for use_vit_merger, run in runs
+ ]
+ )
+
+ def _encode_patches(
+ self, items: List[MultimodalDataItem], use_vit_merger: bool
+ ) -> torch.Tensor:
pixel_values = flatten_nested_list([item.feature for item in items])
tgt_sizes = torch.stack(
flatten_nested_list([item.tgt_size for item in items]), dim=0
@@ -1530,8 +1572,6 @@ class MiniCPMV4_6(MiniCPMBaseModel):
patch_attn_mask.size(2), device=patch_attn_mask.device
).unsqueeze(0) < mask_shapes.unsqueeze(1)
- use_vit_merger = getattr(self.config, "downsample_mode", "16x") != "4x"
-
vision_embedding, tgt_sizes_out = self.vpm(
all_pixel_values.type(dtype),
patch_attention_mask=patch_attn_mask,
@@ -1546,15 +1586,8 @@ class MiniCPMV4_6(MiniCPMBaseModel):
get_video_feature = get_image_feature
def pad_input_ids(self, input_ids: array, image_inputs: MultimodalInputs) -> array:
- im_start_id: int = image_inputs.im_start_id
- im_end_id: int = image_inputs.im_end_id
- slice_start_id: int = image_inputs.slice_start_id
- slice_end_id: int = image_inputs.slice_end_id
-
- media_token_pairs = [(im_start_id, im_end_id), (slice_start_id, slice_end_id)]
- pattern = MultiModalityDataPaddingPatternTokenPairs(
- media_token_pairs, data_start_token_ids=[im_start_id]
- )
+ # Every thumbnail / slice is its own item; pad each span with its own value.
+ pattern = MultiModalityDataPaddingPatternMultimodalTokens()
return pattern.pad_input_tokens(input_ids, image_inputs)
def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
@@ -1606,11 +1639,65 @@ class MiniCPMV4_6(MiniCPMBaseModel):
weight_loader(param, loaded_weight)
+# 4.7 checkpoints keep top-level ``llm`` / ``vpm`` / ``resampler`` / ``vit_merger``
+# names and store connector MLPs as ``nn.Sequential``.
+_V4_7_WEIGHT_PREFIXES = (
+ ("llm.model.", "model.language_model."),
+ ("llm.lm_head.", "lm_head."),
+ ("vit_merger.", "model.vision_tower.vit_merger."),
+ ("vpm.", "model.vision_tower."),
+ ("resampler.", "model.merger."),
+)
+
+
+def _v4_7_to_v4_6_weight_name(name: str) -> str:
+ for old, new in _V4_7_WEIGHT_PREFIXES:
+ if name.startswith(old):
+ name = new + name[len(old) :]
+ break
+ name = re.sub(r"^(model\.merger\.mlp\.\d+)\.mlp\.0\.", r"\1.linear_1.", name)
+ return re.sub(r"^(model\.merger\.mlp\.\d+)\.mlp\.2\.", r"\1.linear_2.", name)
+
+
+class MiniCPMV4_7(MiniCPMV4_6):
+ def __init__(
+ self,
+ config: PretrainedConfig,
+ quant_config: Optional[QuantizationConfig] = None,
+ prefix: str = "",
+ ):
+ super().__init__(config=config, quant_config=quant_config, prefix=prefix)
+ rope_parameters = config.text_config.rope_parameters or {}
+ self.is_mrope_enabled = "mrope_section" in rope_parameters
+
+ def forward(
+ self,
+ input_ids: torch.Tensor,
+ positions: torch.Tensor,
+ forward_batch: ForwardBatch,
+ **kwargs: Any,
+ ) -> torch.Tensor:
+ if self.is_mrope_enabled:
+ positions = forward_batch.mrope_positions
+ return super().forward(
+ input_ids=input_ids,
+ positions=positions,
+ forward_batch=forward_batch,
+ **kwargs,
+ )
+
+ def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
+ return super().load_weights(
+ (_v4_7_to_v4_6_weight_name(name), weight) for name, weight in weights
+ )
+
+
_SUPPORT_VERSION = {
(2, 6): MiniCPMV2_6,
(4, 0): MiniCPMV4_0,
(4, 5): MiniCPMV4_5,
(4, 6): MiniCPMV4_6,
+ (4, 7): MiniCPMV4_7,
}
@@ -1632,8 +1719,8 @@ class MiniCPMV:
@staticmethod
def shared_experts_fusion_disable_reason(hf_config, quant_config):
- # 4.6 nests a Qwen3.5 LLM under ``text_config``; every other version
- # builds a dense LLM, for which the Qwen3.5 gate answers None.
+ # 4.6/4.7 nest a Qwen3.5 config (dense or MoE) under ``text_config``;
+ # older versions are dense, for which the Qwen3.5 gate answers None.
return Qwen3_5ForCausalLM.shared_experts_fusion_disable_reason(
getattr(hf_config, "text_config", hf_config), quant_config
)
@@ -1646,11 +1733,12 @@ class MiniCPMV:
) -> None:
super().__init__()
- # 4.6 carries ``model_type == "minicpmv4_6"`` instead of a numeric
+ # 4.6+ carry their own ``model_type`` instead of a numeric
# ``config.version``; older versionless configs keep the legacy
# ``(2, 6)`` default.
- if getattr(config, "model_type", None) == "minicpmv4_6":
- version = (4, 6)
+ model_type = getattr(config, "model_type", None)
+ if model_type in _MODEL_TYPE_VERSIONS:
+ version = _MODEL_TYPE_VERSIONS[model_type]
elif not hasattr(config, "version"):
version = (2, 6)
else:
@@ -1748,7 +1836,30 @@ class MiniCPMV:
# ``__name__`` — resolves the canonical 4.6 architecture name through
# ``MiniCPMV``'s version-dispatch factory.
class MiniCPMV4_6ForConditionalGeneration(MiniCPMV):
+ # Only for the quant config: its ``ignore`` list names modules in either the
+ # HF or the native 4.7 layout (see _V4_7_WEIGHT_PREFIXES), while
+ # load_weights remaps checkpoint names itself.
+ hf_to_sglang_mapper = WeightsMapper(
+ orig_to_new_substr={"self_attn.out_proj": "self_attn.proj"},
+ orig_to_new_prefix={
+ "model.language_model.": "llm.",
+ "model.vision_tower.": "vpm.",
+ "model.merger.": "resampler.",
+ "llm.model.": "llm.",
+ "llm.lm_head": "lm_head",
+ "vit_merger.": "vpm.vit_merger.",
+ },
+ orig_to_new_suffix={".mlp.0": ".linear_1", ".mlp.2": ".linear_2"},
+ )
+ packed_modules_mapping = Qwen3_5ForCausalLM.packed_modules_mapping
+
+
+class MiniCPMV4_7ForConditionalGeneration(MiniCPMV4_6ForConditionalGeneration):
pass
-EntryClass = [MiniCPMV, MiniCPMV4_6ForConditionalGeneration]
+EntryClass = [
+ MiniCPMV,
+ MiniCPMV4_6ForConditionalGeneration,
+ MiniCPMV4_7ForConditionalGeneration,
+]
diff --git a/python/sglang/srt/multimodal/processors/minicpmv4_6.py b/python/sglang/srt/multimodal/processors/minicpmv4_6.py
index c2a0f67b1b..bf86f833c9 100644
--- a/python/sglang/srt/multimodal/processors/minicpmv4_6.py
+++ b/python/sglang/srt/multimodal/processors/minicpmv4_6.py
@@ -17,6 +17,7 @@ module collapses to a thin wrapper that delegates to it.
from __future__ import annotations
+import hashlib
import math
from itertools import chain
from typing import Any, List, Optional, Sequence, Tuple, Union
@@ -31,6 +32,10 @@ from sglang.srt.managers.schedule_batch import (
MultimodalProcessorOutput,
)
from sglang.srt.models.minicpmv import MiniCPMV4_6ForConditionalGeneration
+from sglang.srt.multimodal.cache import (
+ CONTENT_HASH_PREFIX,
+ resolve_multimodal_item_hash,
+)
from sglang.srt.multimodal.processors.base_processor import (
BaseMultimodalProcessor,
MultimodalSpecialTokens,
@@ -127,16 +132,16 @@ def _reshape_by_patch(image: torch.Tensor, patch_size: int) -> torch.Tensor:
return patches
-def _flatten_patches(
- per_item_pv: List[List[torch.Tensor]],
- per_item_ts: List[List[List[int]]],
-) -> Tuple[List[torch.Tensor], List[torch.Tensor]]:
- """Per-item per-patch -> flat per-patch (source first, slices row-major)."""
- flat_pv = list(chain.from_iterable(per_item_pv))
- flat_ts = [
- torch.tensor(ts, dtype=torch.int32) for ts in chain.from_iterable(per_item_ts)
- ]
- return flat_pv, flat_ts
+def _layout_namespace(
+ downsample_mode: str,
+ tgt_size: List[int],
+ media_tgt_sizes: List[List[int]],
+ media_grids: List[List[int]],
+) -> str:
+ # Pixels alone omit the layout: patches of different shape can pack to the
+ # same tensor while their embeddings and M-RoPE positions differ.
+ layout = f"{downsample_mode}|{tgt_size}|{media_tgt_sizes}|{media_grids}"
+ return CONTENT_HASH_PREFIX + hashlib.sha256(layout.encode()).hexdigest()
class MiniCPMV4_6ImageProcessor:
@@ -231,6 +236,14 @@ class MiniCPMV4_6ImageProcessor:
min_error = error
return best_grid
+ @property
+ def use_vit_merger(self) -> bool:
+ return self.downsample_mode != "4x"
+
+ @property
+ def pad_divisor(self) -> int:
+ return 16 if self.use_vit_merger else 4
+
def _normalize(self, t: torch.Tensor) -> torch.Tensor:
t = t * self.rescale_factor
return (t - self.image_mean.to(t.dtype)) / self.image_std.to(t.dtype)
@@ -305,6 +318,7 @@ class MiniCPMV4_6MultimodalProcessor(BaseMultimodalProcessor):
models = [MiniCPMV4_6ForConditionalGeneration]
support_dynamic_frame_expansion = False
gpu_image_decode = False
+ video_image_id = True
def __init__(self, hf_config, server_args, _processor, *args, **kwargs):
super().__init__(hf_config, server_args, _processor, *args, **kwargs)
@@ -353,8 +367,6 @@ class MiniCPMV4_6MultimodalProcessor(BaseMultimodalProcessor):
self.slice_start_id = self._token_id(self.slice_start_token)
self.slice_end_id = self._token_id(self.slice_end_token)
- self.pad_divisor = 16 if downsample_mode != "4x" else 4
-
self.mm_tokens = MultimodalSpecialTokens(
image_token=self.image_token,
image_token_id=self.image_token_id,
@@ -375,19 +387,20 @@ class MiniCPMV4_6MultimodalProcessor(BaseMultimodalProcessor):
self,
tgt_sizes: List[List[int]],
grid: List[int],
+ pad_divisor: int,
) -> str:
"""``...`` (+ optional ``...`` rows) for
one image or video frame; inner pads are ``_PAD_PLACEHOLDER`` (caller
swaps back after splicing).
"""
h0, w0 = tgt_sizes[0]
- n_src = (h0 * w0) // self.pad_divisor
+ n_src = (h0 * w0) // pad_divisor
out = self.image_start_token + _PAD_PLACEHOLDER * n_src + self.image_end_token
if len(tgt_sizes) > 1 and grid and grid[0] > 0 and grid[1] > 0:
grid_y, grid_x = int(grid[0]), int(grid[1])
h_s, w_s = tgt_sizes[1]
- n_slice = (h_s * w_s) // self.pad_divisor
+ n_slice = (h_s * w_s) // pad_divisor
slice_chunk = (
self.slice_start_token
+ _PAD_PLACEHOLDER * n_slice
@@ -401,9 +414,13 @@ class MiniCPMV4_6MultimodalProcessor(BaseMultimodalProcessor):
self,
index: int,
frames: Sequence[Tuple[List[List[int]], List[int]]],
+ pad_divisor: int,
+ image_id: bool = True,
) -> str:
"""One image or one video. Image is a single-frame video."""
- body = "".join(self._expand_frame(ts, grid) for ts, grid in frames)
+ body = "".join(self._expand_frame(ts, grid, pad_divisor) for ts, grid in frames)
+ if not image_id:
+ return body
return f"{self.image_id_start_token}{index}{self.image_id_end_token}" + body
async def process_mm_data_async(
@@ -433,19 +450,28 @@ class MiniCPMV4_6MultimodalProcessor(BaseMultimodalProcessor):
images = base.images or []
videos = base.videos or []
+ image_processor = self._image_processor_for(request_obj)
+ pad_divisor = image_processor.pad_divisor
+
# Image: one "frame" per image. Video: per-frame nesting kept so each
# frame becomes its own ``...`` block in the expansion.
- img_per_pv, img_per_ts, img_grids = self._preprocess_images(images)
- vid_per_pv, vid_per_ts, vid_grids = self._preprocess_videos(videos)
+ img_per_pv, img_per_ts, img_grids = self._preprocess_images(
+ images, image_processor
+ )
+ vid_per_pv, vid_per_ts, vid_grids = self._preprocess_videos(
+ videos, image_processor
+ )
prompt = self._splice_expansions(
prompt,
(
- self._expand_media(i, [(ts, gd)])
+ self._expand_media(i, [(ts, gd)], pad_divisor)
for i, (ts, gd) in enumerate(zip(img_per_ts, img_grids))
),
(
- self._expand_media(i, list(zip(fts, fgd)))
+ self._expand_media(
+ i, list(zip(fts, fgd)), pad_divisor, self.video_image_id
+ )
for i, (fts, fgd) in enumerate(zip(vid_per_ts, vid_grids))
),
)
@@ -462,20 +488,29 @@ class MiniCPMV4_6MultimodalProcessor(BaseMultimodalProcessor):
self._build_items(
input_ids_tensor,
self.image_token_id,
- _flatten_patches(img_per_pv, img_per_ts),
+ [
+ (pvs, tss, [grid])
+ for pvs, tss, grid in zip(img_per_pv, img_per_ts, img_grids)
+ ],
Modality.IMAGE,
+ image_processor,
)
)
- # Video: extra ``per-frame -> per-patch`` nesting; pre-flatten one
- # level so ``_flatten_patches`` sees the same shape as image.
- vid_pv_flat = [list(chain.from_iterable(v)) for v in vid_per_pv]
- vid_ts_flat = [list(chain.from_iterable(v)) for v in vid_per_ts]
+ # Video: one media unit per video, its frames' patches flattened in order.
mm_items.extend(
self._build_items(
input_ids_tensor,
self.video_token_id,
- _flatten_patches(vid_pv_flat, vid_ts_flat),
+ [
+ (
+ list(chain.from_iterable(pvs)),
+ list(chain.from_iterable(tss)),
+ grids,
+ )
+ for pvs, tss, grids in zip(vid_per_pv, vid_per_ts, vid_grids)
+ ],
Modality.VIDEO,
+ image_processor,
)
)
@@ -483,24 +518,28 @@ class MiniCPMV4_6MultimodalProcessor(BaseMultimodalProcessor):
mm_items=mm_items,
input_ids=input_ids,
im_token_id=self.image_token_id,
+ video_token_id=self.video_token_id,
im_start_id=self.image_start_id,
im_end_id=self.image_end_id,
slice_start_id=self.slice_start_id,
slice_end_id=self.slice_end_id,
)
- def _preprocess_images(self, images):
+ def _image_processor_for(self, request_obj) -> MiniCPMV4_6ImageProcessor:
+ return self.image_processor
+
+ def _preprocess_images(self, images, image_processor):
if not images:
return [], [], []
- out = self.image_processor.preprocess(images)
+ out = image_processor.preprocess(images)
return out["pixel_values"], out["tgt_sizes"], out["grids"]
- def _preprocess_videos(self, videos):
+ def _preprocess_videos(self, videos, image_processor):
per_video_pv: List[List[List[torch.Tensor]]] = []
per_video_ts: List[List[List[List[int]]]] = []
per_video_grids: List[List[List[int]]] = []
for frames in videos:
- out = self.image_processor.preprocess(list(frames))
+ out = image_processor.preprocess(list(frames))
per_video_pv.append(out["pixel_values"])
per_video_ts.append(out["tgt_sizes"])
per_video_grids.append(out["grids"])
@@ -527,22 +566,33 @@ class MiniCPMV4_6MultimodalProcessor(BaseMultimodalProcessor):
self,
input_ids: torch.Tensor,
pad_token_id: int,
- flat: Tuple[List[torch.Tensor], List[torch.Tensor]],
+ media: List[Tuple[List[torch.Tensor], List[List[int]], List[List[int]]]],
modality: Modality,
+ image_processor: MiniCPMV4_6ImageProcessor,
) -> List[MultimodalDataItem]:
- flat_pv, flat_ts = flat
+ """``media``: per image / video, its patches' pixel values, tgt sizes
+ and slice grids (source patch first, slices row-major)."""
+ patches = [
+ (pv, ts, _layout_namespace(image_processor.downsample_mode, ts, tss, grids))
+ for pvs, tss, grids in media
+ for pv, ts in zip(pvs, tss)
+ ]
runs = self.get_mm_items_offset(input_ids, pad_token_id)
- if len(runs) != len(flat_pv):
+ if len(runs) != len(patches):
raise RuntimeError(
f"[minicpmv4_6] {modality} pad run / feature count mismatch: "
- f"{len(runs)} runs vs {len(flat_pv)} patches"
+ f"{len(runs)} runs vs {len(patches)} patches"
)
return [
MultimodalDataItem(
feature=[pv],
+ hash=resolve_multimodal_item_hash(feature=[pv], namespace=namespace),
offsets=[run],
- model_specific_data={"tgt_size": [ts]},
+ model_specific_data={
+ "tgt_size": [torch.tensor(ts, dtype=torch.int32)],
+ "use_vit_merger": image_processor.use_vit_merger,
+ },
modality=modality,
)
- for run, pv, ts in zip(runs, flat_pv, flat_ts)
+ for run, (pv, ts, namespace) in zip(runs, patches)
]
diff --git a/python/sglang/srt/multimodal/processors/minicpmv4_7.py b/python/sglang/srt/multimodal/processors/minicpmv4_7.py
new file mode 100644
index 0000000000..5b1e2e6b56
--- /dev/null
+++ b/python/sglang/srt/multimodal/processors/minicpmv4_7.py
@@ -0,0 +1,347 @@
+# Copyright 2026 The SGLang team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+"""SGLang multimodal processor for MiniCPM-V 4.7.
+
+The canvas M-RoPE layout ports the checkpoints' reference
+``mrope_minicpmv4_7.py`` and must assign exactly the same positions.
+"""
+
+from __future__ import annotations
+
+import logging
+import math
+from typing import List, Optional, Tuple
+
+import torch
+
+from sglang.srt.models.minicpmv import MiniCPMV4_7ForConditionalGeneration
+from sglang.srt.multimodal.processors.minicpmv4_6 import (
+ MiniCPMV4_6ImageProcessor,
+ MiniCPMV4_6MultimodalProcessor,
+)
+
+logger = logging.getLogger(__name__)
+
+_SCOPED_KWARGS_KEYS = ("images_kwargs", "videos_kwargs", "audio_kwargs")
+_STRUCTURAL_KEYS = ("im_start_id", "im_end_id", "slice_start_id", "slice_end_id")
+
+# (start, end, index into target_sizes) of one run of visual tokens.
+_Span = Tuple[int, int, int]
+_Frame = Tuple[_Span, List[_Span]]
+
+
+def _parse_mm_processor_kwargs(raw) -> dict:
+ """Flatten HF-style ``images_kwargs`` / ``videos_kwargs``; top-level keys win."""
+ if not isinstance(raw, dict):
+ return {}
+ flat = {}
+ for key in _SCOPED_KWARGS_KEYS:
+ if isinstance(raw.get(key), dict):
+ flat.update(raw[key])
+ flat.update({k: v for k, v in raw.items() if k not in _SCOPED_KWARGS_KEYS})
+ return flat
+
+
+def _llm_grid(n_tokens: int, vit_h: int, vit_w: int) -> Tuple[int, int]:
+ """LLM-token grid for ``n_tokens`` merged from a ``vit_h x vit_w`` patch grid."""
+ if n_tokens <= 0 or vit_h <= 0 or vit_w <= 0:
+ return 0, 0
+ vit_total = vit_h * vit_w
+ if vit_total == n_tokens:
+ return vit_h, vit_w
+ if vit_total < n_tokens:
+ return 0, 0
+ factor = int(round(math.sqrt(vit_total / n_tokens)))
+ if factor > 0 and vit_h % factor == 0 and vit_w % factor == 0:
+ height, width = vit_h // factor, vit_w // factor
+ if height * width == n_tokens:
+ return height, width
+ ratio = vit_h / vit_w
+ width = max(1, int(round(math.sqrt(n_tokens / ratio))))
+ height = n_tokens // width
+ if height * width == n_tokens and height > 0:
+ return height, width
+ return 0, 0
+
+
+def build_image_bounds(input_ids: torch.Tensor, token_ids: dict) -> torch.Tensor:
+ """``[start, end)`` of every thumbnail / slice token run, shape (n, 2)."""
+ start_ids = [token_ids[k] for k in ("im_start_id", "slice_start_id")]
+ end_ids = [token_ids[k] for k in ("im_end_id", "slice_end_id")]
+ starts = torch.isin(input_ids, torch.tensor(start_ids)).nonzero().flatten() + 1
+ ends = torch.isin(input_ids, torch.tensor(end_ids)).nonzero().flatten()
+ n = min(len(starts), len(ends))
+ return torch.stack([starts[:n], ends[:n]], dim=-1)
+
+
+def _frame_end(ids: List[int], frame: _Frame, structural: set, limit: int) -> int:
+ thumbnail, slices = frame
+ end = slices[-1][1] if slices else thumbnail[1]
+ while end < limit and ids[end] in structural:
+ end += 1
+ return min(end, limit)
+
+
+def _group_frames(
+ ids: List[int], bounds: List[List[int]], token_ids: dict, structural: set
+) -> List[List[_Frame]]:
+ """Pair each thumbnail with its slices, then merge frames separated only by
+ structural / newline tokens into one video group."""
+ frames: List[_Frame] = []
+ for index, (start, end) in enumerate(bounds):
+ if frames and ids[start - 1] == token_ids["slice_start_id"]:
+ frames[-1][1].append((start, end, index))
+ else:
+ frames.append(((start, end, index), []))
+
+ gap_ok = structural | {token_ids["newline_id"]}
+ groups = [[frames[0]]]
+ for prev, frame in zip(frames, frames[1:]):
+ gap = range(_frame_end(ids, prev, structural, len(ids)), frame[0][0] - 1)
+ if all(ids[p] in gap_ok for p in gap):
+ groups[-1].append(frame)
+ else:
+ groups.append([frame])
+ return groups
+
+
+def _fill_text(positions: torch.Tensor, start: int, end: int, pos: int) -> int:
+ if end <= start:
+ return pos
+ positions[:, start:end] = torch.arange(pos, pos + end - start)
+ return pos + end - start
+
+
+def _fill_grid(
+ positions: torch.Tensor,
+ start: int,
+ base: int,
+ rows: torch.Tensor,
+ cols: torch.Tensor,
+) -> None:
+ end = start + len(rows) * len(cols)
+ positions[0, start:end] = base
+ positions[1, start:end] = rows.repeat_interleave(len(cols)) + base
+ positions[2, start:end] = cols.repeat(len(rows)) + base
+
+
+def _place_frame(
+ positions: torch.Tensor,
+ frame: _Frame,
+ frame_end: int,
+ base: int,
+ sizes: List[List[int]],
+) -> int:
+ """Lay one thumbnail and its slice grid out on a shared canvas at ``base``;
+ returns the first position after the canvas."""
+ (t_start, t_end, t_index), slices = frame
+ im_start = t_start - 1
+ n_thumb = t_end - t_start
+ th, tw = _llm_grid(n_thumb, *sizes[t_index])
+
+ if slices:
+ cols = len(slices)
+ for k in range(len(slices) - 1):
+ if slices[k + 1][0] - slices[k][1] > 2:
+ cols = k + 1
+ break
+ rows = len(slices) // cols
+ if rows * cols != len(slices):
+ rows, cols = 1, len(slices)
+ s0_start, s0_end, s0_index = slices[0]
+ sh, sw = _llm_grid(s0_end - s0_start, *sizes[s0_index])
+ canvas_h, canvas_w = rows * sh, cols * sw
+ else:
+ canvas_h, canvas_w = max(th, 1), max(tw, 1)
+
+ positions[:, im_start:frame_end] = base
+ positions[1:, im_start] = max(base - 1, 0)
+ if t_end < frame_end:
+ positions[1, t_end] = base + canvas_h
+ positions[2, t_end] = base + canvas_w
+
+ if th > 0:
+ if slices and canvas_h > 0 and canvas_w > 0:
+ # The thumbnail is stretched over the full slice canvas.
+ t_rows = torch.linspace(0, canvas_h - 1, th).round().long()
+ t_cols = torch.linspace(0, canvas_w - 1, tw).round().long()
+ else:
+ t_rows, t_cols = torch.arange(th), torch.arange(tw)
+ _fill_grid(positions, t_start, base, t_rows, t_cols)
+ else:
+ _fill_text(positions, t_start, t_end, base)
+
+ for k, (s_start, s_end, s_index) in enumerate(slices):
+ sh_k, sw_k = _llm_grid(s_end - s_start, *sizes[s_index])
+ h_off, w_off = (k // cols) * sh, (k % cols) * sw
+ positions[1, s_start - 1] = base + h_off
+ positions[2, s_start - 1] = base + w_off
+ if s_end < frame_end:
+ positions[1, s_end] = base + h_off + max(sh_k - 1, 0)
+ positions[2, s_end] = base + w_off + max(sw_k - 1, 0)
+ if sh_k > 0:
+ rows_k = torch.arange(sh_k) + h_off
+ cols_k = torch.arange(sw_k) + w_off
+ _fill_grid(positions, s_start, base, rows_k, cols_k)
+ else:
+ _fill_text(positions, s_start, s_end, base)
+
+ # Row-separating newlines sit one column past the canvas, on the row just ended.
+ for k in range(len(slices) - 1):
+ gap_start, gap_end = slices[k][1] + 1, slices[k + 1][0] - 1
+ if gap_end > gap_start:
+ positions[1, gap_start:gap_end] = base + (k // cols + 1) * sh - 1
+ positions[2, gap_start:gap_end] = base + cols * sw
+
+ return base + max(canvas_h, canvas_w) + 1
+
+
+def compute_canvas_positions(
+ input_ids: torch.Tensor,
+ image_bounds: torch.Tensor,
+ target_sizes: List[List[int]],
+ token_ids: dict,
+) -> torch.Tensor:
+ """(3, seq_len) ``[temporal, height, width]`` canvas M-RoPE positions."""
+ ids = input_ids.tolist()
+ seq_len = len(ids)
+ positions = torch.arange(seq_len).expand(3, -1).clone()
+ if len(image_bounds) == 0:
+ return positions
+
+ structural = {token_ids[k] for k in _STRUCTURAL_KEYS}
+ pos = cursor = 0
+ for frames in _group_frames(ids, image_bounds.tolist(), token_ids, structural):
+ group_start = frames[0][0][0] - 1
+ group_end = _frame_end(ids, frames[-1], structural, seq_len)
+ pos = _fill_text(positions, cursor, group_start, pos)
+ frame_cursor = group_start
+ for frame in frames:
+ im_start = frame[0][0] - 1
+ if im_start > frame_cursor:
+ # The extra step keeps the next frame's halo off the gap.
+ pos = _fill_text(positions, frame_cursor, im_start, pos) + 1
+ frame_end = _frame_end(ids, frame, structural, group_end)
+ pos = _place_frame(positions, frame, frame_end, pos, target_sizes)
+ frame_cursor = frame_end
+ cursor = group_end
+ _fill_text(positions, cursor, seq_len, pos)
+ return positions
+
+
+def _sequential_positions(seq_len: int) -> torch.Tensor:
+ return torch.arange(seq_len).expand(3, -1)
+
+
+def _rope_delta(positions: torch.Tensor) -> torch.Tensor:
+ """Decode offset: the first generated token sits one past the last prompt position."""
+ delta = int(positions.amax()) + 1 - positions.shape[1]
+ return torch.tensor([[delta]], dtype=torch.int64)
+
+
+class MiniCPMV4_7MultimodalProcessor(MiniCPMV4_6MultimodalProcessor):
+ models = [MiniCPMV4_7ForConditionalGeneration]
+ # 4.7 was trained without ```` in front of videos.
+ video_image_id = False
+
+ def __init__(self, hf_config, server_args, _processor, *args, **kwargs):
+ super().__init__(hf_config, server_args, _processor, *args, **kwargs)
+ self.canvas_token_ids = self._resolve_canvas_token_ids(hf_config)
+
+ def _image_processor_for(self, request_obj) -> MiniCPMV4_6ImageProcessor:
+ kwargs = _parse_mm_processor_kwargs(
+ getattr(request_obj, "mm_processor_kwargs", None)
+ )
+ default = self.image_processor
+
+ downsample_mode = kwargs.get("downsample_mode") or default.downsample_mode
+ if downsample_mode not in ("4x", "16x"):
+ logger.warning("Ignoring unknown downsample_mode %r", downsample_mode)
+ downsample_mode = default.downsample_mode
+ max_slice_nums = kwargs.get("max_slice_nums")
+ if max_slice_nums is None:
+ max_slice_nums = default.max_slice_nums
+ try:
+ max_slice_nums = int(max_slice_nums)
+ except (TypeError, ValueError):
+ logger.warning("Ignoring non-numeric max_slice_nums %r", max_slice_nums)
+ max_slice_nums = default.max_slice_nums
+
+ if (downsample_mode, max_slice_nums) == (
+ default.downsample_mode,
+ default.max_slice_nums,
+ ):
+ return default
+ return MiniCPMV4_6ImageProcessor(
+ max_slice_nums=max_slice_nums,
+ scale_resolution=default.scale_resolution,
+ patch_size=default.patch_size,
+ downsample_mode=downsample_mode,
+ )
+
+ def _resolve_canvas_token_ids(self, hf_config) -> Optional[dict]:
+ mrope_mode = str(getattr(hf_config, "mrope_mode", "")).strip().lower()
+ if (
+ not getattr(hf_config, "uses_mrope_canvas", False)
+ and mrope_mode != "canvas"
+ ):
+ return None
+
+ # Byte-level BPE vocabularies store "\n" as "\u010a", so encode it.
+ newline_ids = self.tokenizer.encode("\n", add_special_tokens=False)
+ from_tokenizer = {
+ "im_start_id": self.image_start_id,
+ "im_end_id": self.image_end_id,
+ "slice_start_id": self.slice_start_id,
+ "slice_end_id": self.slice_end_id,
+ "newline_id": newline_ids[0] if len(newline_ids) == 1 else None,
+ }
+ token_ids = {
+ "im_start_id": getattr(hf_config, "image_start_id", None),
+ "im_end_id": getattr(hf_config, "image_end_id", None),
+ "slice_start_id": getattr(hf_config, "slice_start_id", None),
+ "slice_end_id": getattr(hf_config, "slice_end_id", None),
+ "newline_id": getattr(hf_config, "newline_id", None),
+ }
+ token_ids = {
+ k: from_tokenizer[k] if v is None else v for k, v in token_ids.items()
+ }
+ if None in token_ids.values():
+ logger.warning("Canvas M-RoPE disabled: token ids missing (%s)", token_ids)
+ return None
+ return {k: int(v) for k, v in token_ids.items()}
+
+ def compute_mrope_positions(self, input_ids, mm_items):
+ ids = torch.tensor(list(input_ids), dtype=torch.long)
+ positions = self._canvas_positions(ids, mm_items)
+ # The Qwen3.5 backbone reads mrope_positions for every multimodal request.
+ if positions is None:
+ positions = _sequential_positions(len(ids))
+ return positions, _rope_delta(positions)
+
+ def _canvas_positions(self, ids, mm_items) -> Optional[torch.Tensor]:
+ if self.canvas_token_ids is None:
+ return None
+ # Items are appended images-first; the canvas walk needs them in prompt order.
+ ordered = sorted(mm_items, key=lambda item: item.offsets[0][0])
+ target_sizes = [torch.as_tensor(item.tgt_size[0]).tolist() for item in ordered]
+ image_bounds = build_image_bounds(ids, self.canvas_token_ids)
+ if len(image_bounds) != len(target_sizes):
+ logger.warning(
+ "Canvas M-RoPE: %d image bounds vs %d patch grids; "
+ "using sequential positions",
+ len(image_bounds),
+ len(target_sizes),
+ )
+ return None
+ return compute_canvas_positions(
+ ids, image_bounds, target_sizes, self.canvas_token_ids
+ )
+
+
+__all__ = ["MiniCPMV4_7MultimodalProcessor"]
diff --git a/python/sglang/srt/parser/conversation.py b/python/sglang/srt/parser/conversation.py
index 585ecdca55..dd6df9572e 100644
--- a/python/sglang/srt/parser/conversation.py
+++ b/python/sglang/srt/parser/conversation.py
@@ -1196,11 +1196,11 @@ def match_minicpm(model_path: str):
# must NOT fall back to the legacy `minicpmv` conv template (which encodes
# the old `(./)` placeholder used by 2.x/4.0/4.5).
model_type = get_model_type(model_path)
- if model_type == "minicpmv4_6":
+ if model_type in ("minicpmv4_6", "minicpmv4_7"):
return None
# For HF-hub paths (where config.json isn't on local disk yet), fall back
# to a path-version check: exclude 4.6 and later, match only legacy 2.x/4.0/4.5.
- if re.search(r"minicpm-(v|o)-4[._]6", model_path, re.IGNORECASE):
+ if re.search(r"minicpm-(v|o)-4[._][67]", model_path, re.IGNORECASE):
return None
match = re.search(r"minicpm-(v|o)", model_path, re.IGNORECASE)
if match:
diff --git a/python/sglang/srt/speculative/dspark_components/dspark_config.py b/python/sglang/srt/speculative/dspark_components/dspark_config.py
index 50909f9f7a..48143d3fc7 100644
--- a/python/sglang/srt/speculative/dspark_components/dspark_config.py
+++ b/python/sglang/srt/speculative/dspark_components/dspark_config.py
@@ -193,8 +193,8 @@ def _get_text_config(config: Any) -> Any:
return config
-def _get_dspark_config(config: Any) -> dict:
- cfg = _cfg_get(config, "dspark_config", None)
+def _get_nested_config(config: Any, key: str) -> dict:
+ cfg = _cfg_get(config, key, None)
if cfg is None:
return {}
if isinstance(cfg, dict):
@@ -205,6 +205,14 @@ def _get_dspark_config(config: Any) -> dict:
return {}
+def _get_dspark_config(config: Any) -> dict:
+ # Published DSpark drafts nest their fields under dflash_config.
+ return {
+ **_get_nested_config(config, "dflash_config"),
+ **_get_nested_config(config, "dspark_config"),
+ }
+
+
def parse_dspark_draft_config(*, draft_hf_config: Any) -> DSparkDraftConfig:
base = parse_dflash_draft_config(draft_hf_config=draft_hf_config)
diff --git a/python/sglang/srt/utils/hf_transformers/common.py b/python/sglang/srt/utils/hf_transformers/common.py
index 5c90ddfd93..6d98a22bad 100644
--- a/python/sglang/srt/utils/hf_transformers/common.py
+++ b/python/sglang/srt/utils/hf_transformers/common.py
@@ -69,6 +69,7 @@ from sglang.srt.configs import (
MiniCPMHybridConfig,
MiniCPMV4_6Config,
MiniCPMV4_6VisionConfig,
+ MiniCPMV4_7Config,
MiniMaxM3VLConfig,
MultiModalityConfig,
MuseGlimmerAssistantConfig,
@@ -176,6 +177,7 @@ _CONFIG_REGISTRY: Dict[str, Type[PretrainedConfig]] = {
MiniCPMHybridConfig,
MiniCPMV4_6Config,
MiniCPMV4_6VisionConfig,
+ MiniCPMV4_7Config,
InklingModelConfig,
InklingAudioConfig,
InklingVisionConfig,
diff --git a/test/registered/unit/api/v1/test_parity.py b/test/registered/unit/api/v1/test_parity.py
index 4058fb4bb2..98bbc82c10 100644
--- a/test/registered/unit/api/v1/test_parity.py
+++ b/test/registered/unit/api/v1/test_parity.py
@@ -62,6 +62,7 @@ PYTHON_ONLY_REQUEST_FIELDS = frozenset(
"max_thinking_tokens",
"min_dynamic_patch",
"mm_content_hashes",
+ "mm_processor_kwargs",
"modalities",
"multi_item_delimiter_indices",
"need_wait_for_mm_inputs",
diff --git a/test/registered/unit/multimodal/test_minicpmv4_7_canvas_mrope.py b/test/registered/unit/multimodal/test_minicpmv4_7_canvas_mrope.py
new file mode 100644
index 0000000000..cc50d661e2
--- /dev/null
+++ b/test/registered/unit/multimodal/test_minicpmv4_7_canvas_mrope.py
@@ -0,0 +1,65 @@
+"""Canvas M-RoPE layout for MiniCPM-V 4.7 (CPU only).
+
+The expected positions come from the checkpoints' reference
+``mrope_minicpmv4_7.py`` and pin the port to it.
+"""
+
+import unittest
+
+import torch
+
+from sglang.srt.multimodal.processors.minicpmv4_7 import (
+ build_image_bounds,
+ compute_canvas_positions,
+)
+from sglang.test.ci.ci_register import register_cpu_ci
+from sglang.test.test_utils import CustomTestCase
+
+register_cpu_ci(est_time=5, suite="base-a-test-cpu")
+
+# Synthetic ids: the layout code only ever compares them for equality.
+IM_START, IM_END, SLICE_START, SLICE_END, NEWLINE, PAD = 900, 901, 902, 903, 904, 905
+SPECIAL_IDS = {
+ "im_start_id": IM_START,
+ "im_end_id": IM_END,
+ "slice_start_id": SLICE_START,
+ "slice_end_id": SLICE_END,
+ "newline_id": NEWLINE,
+}
+
+
+def _canvas(input_ids, target_sizes):
+ ids = torch.tensor(input_ids, dtype=torch.long)
+ return compute_canvas_positions(
+ ids, build_image_bounds(ids, SPECIAL_IDS), target_sizes, SPECIAL_IDS
+ )
+
+
+class TestMiniCPMV4_7CanvasMRoPE(CustomTestCase):
+ def test_thumbnail_grid_places_rows_and_columns(self):
+ # 16 visual tokens on a 4x4 grid: halo sits one position below
+ # the base, one past the canvas.
+ positions = _canvas(
+ [10, 11, IM_START] + [PAD] * 16 + [IM_END, 12, 13], [(4, 4)]
+ )
+
+ self.assertEqual(positions[0].tolist(), [0, 1, 2] + [2] * 16 + [2, 7, 8])
+ self.assertEqual(
+ positions[1].tolist(),
+ [0, 1, 1] + [2, 2, 2, 2, 3, 3, 3, 3, 4, 4, 4, 4, 5, 5, 5, 5] + [6, 7, 8],
+ )
+ self.assertEqual(
+ positions[2].tolist(),
+ [0, 1, 1] + [2, 3, 4, 5, 2, 3, 4, 5, 2, 3, 4, 5, 2, 3, 4, 5] + [6, 7, 8],
+ )
+
+ def test_tightly_adjacent_frames_form_one_video_group(self):
+ frame = [IM_START] + [PAD] * 16 + [IM_END]
+ positions = _canvas([10, 11] + frame * 3 + [12, 13], [(4, 4)] * 3)
+
+ # Each frame advances its own base instead of sharing the first one's.
+ self.assertTrue((positions[0][20:36] > positions[0][3:19].max()).all())
+
+
+if __name__ == "__main__":
+ unittest.main()