diff --git a/python/sglang/srt/arg_groups/model_overrides/minicpmv.py b/python/sglang/srt/arg_groups/model_overrides/minicpmv.py index 9fe11f0b10..a28de2e0a5 100644 --- a/python/sglang/srt/arg_groups/model_overrides/minicpmv.py +++ b/python/sglang/srt/arg_groups/model_overrides/minicpmv.py @@ -10,6 +10,7 @@ from sglang.srt.runtime_context import get_platform @_register_for("MiniCPMV4_6ForConditionalGeneration") +@_register_for("MiniCPMV4_7ForConditionalGeneration") def _minicpm_v4_6_overrides(server_args: Any, hf_config: Any) -> dict: cfg = resolving_view(server_args) if get_platform().is_sm100 and cfg.attention_backend is None: diff --git a/python/sglang/srt/arg_groups/overrides.py b/python/sglang/srt/arg_groups/overrides.py index ac79251c93..b30360d623 100644 --- a/python/sglang/srt/arg_groups/overrides.py +++ b/python/sglang/srt/arg_groups/overrides.py @@ -362,6 +362,7 @@ _MAMBA_RADIX_CACHE_ARCHS = frozenset( "Qwen3_5ForCausalLM", "Qwen4ExpForConditionalGeneration", "MiniCPMV4_6ForConditionalGeneration", + "MiniCPMV4_7ForConditionalGeneration", "NemotronHForCausalLM", "NemotronHPuzzleForCausalLM", "NemotronH_Omni_Reasoning_V3", @@ -390,6 +391,7 @@ _MAMBA_EXTRA_BUFFER_ARCHS = frozenset( "Qwen4ExpForConditionalGeneration", "InternS2PreviewForConditionalGeneration", "MiniCPMV4_6ForConditionalGeneration", + "MiniCPMV4_7ForConditionalGeneration", "BailingMoeV2_5ForCausalLM", "BailingMoeV3ForCausalLM", "BailingMoeV3VLForConditionalGeneration", diff --git a/python/sglang/srt/configs/__init__.py b/python/sglang/srt/configs/__init__.py index 06239f17ee..69f21c1ae6 100644 --- a/python/sglang/srt/configs/__init__.py +++ b/python/sglang/srt/configs/__init__.py @@ -56,6 +56,7 @@ from sglang.srt.configs.mamba import FalconMambaConfig, MambaConfig from sglang.srt.configs.mamba2 import Mamba2Config from sglang.srt.configs.minicpm import MiniCPMHybridConfig from sglang.srt.configs.minicpmv4_6 import MiniCPMV4_6Config, MiniCPMV4_6VisionConfig +from sglang.srt.configs.minicpmv4_7 import MiniCPMV4_7Config from sglang.srt.configs.minimax_vl import MiniMaxM3VLConfig from sglang.srt.configs.muse_glimmer import ( MuseGlimmerAssistantConfig, @@ -150,6 +151,7 @@ __all__ = [ "LocateAnythingConfig", "MiniCPMV4_6Config", "MiniCPMV4_6VisionConfig", + "MiniCPMV4_7Config", "NemotronHConfig", "NemotronHPuzzleConfig", "NemotronH_Nano_VL_V2_Config", diff --git a/python/sglang/srt/configs/minicpmv4_6.py b/python/sglang/srt/configs/minicpmv4_6.py index 472224a892..69dae04518 100644 --- a/python/sglang/srt/configs/minicpmv4_6.py +++ b/python/sglang/srt/configs/minicpmv4_6.py @@ -17,7 +17,7 @@ from typing import Any, Dict, Optional, Union from transformers import AutoConfig, PretrainedConfig from transformers.models.auto import CONFIG_MAPPING -from sglang.srt.configs.qwen3_5 import Qwen3_5TextConfig +from sglang.srt.configs.qwen3_5 import Qwen3_5MoeTextConfig, Qwen3_5TextConfig class MiniCPMV4_6VisionConfig(PretrainedConfig): @@ -54,15 +54,17 @@ class MiniCPMV4_6VisionConfig(PretrainedConfig): def _resolve_text_config_class(model_type: Optional[str]) -> type: - """``model_type`` -> registered config class. sglang's ``Qwen3_5TextConfig`` - wins over the stock entry when both exist (it carries ``layers_block_type`` - etc. that the model code reads); ``AutoConfig.register`` doesn't replace - existing entries so we have to short-circuit here. Note that + """``model_type`` -> registered config class. sglang's Qwen3.5 dense / MoE + text configs win over the stock entries when both exist (they carry + ``layers_block_type`` etc. that the model code reads); + ``AutoConfig.register`` doesn't replace existing entries so we have to + short-circuit here. Note that ``CONFIG_MAPPING.get`` returns ``None`` even on hit — go through ``__getitem__`` to trigger the lazy class import. """ - if model_type == Qwen3_5TextConfig.model_type: - return Qwen3_5TextConfig + for cls in (Qwen3_5TextConfig, Qwen3_5MoeTextConfig): + if model_type == cls.model_type: + return cls if model_type and model_type in CONFIG_MAPPING: return CONFIG_MAPPING[model_type] raise KeyError(f"Unknown text_config model_type: {model_type!r}") diff --git a/python/sglang/srt/configs/minicpmv4_7.py b/python/sglang/srt/configs/minicpmv4_7.py new file mode 100644 index 0000000000..2a52a73412 --- /dev/null +++ b/python/sglang/srt/configs/minicpmv4_7.py @@ -0,0 +1,22 @@ +# Copyright 2026 The SGLang team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +from typing import Any + +from sglang.srt.configs.minicpmv4_6 import MiniCPMV4_6Config + + +class MiniCPMV4_7Config(MiniCPMV4_6Config): + model_type = "minicpmv4_7" + + # transformers 5 wraps config subclasses in ``@dataclass(kw_only=True)``; + # its generated ``__init__`` would skip ``MiniCPMV4_6Config.__init__``. + def __init__(self, **kwargs: Any) -> None: + super().__init__(**kwargs) + + +__all__ = ["MiniCPMV4_7Config"] diff --git a/python/sglang/srt/entrypoints/openai/protocol.py b/python/sglang/srt/entrypoints/openai/protocol.py index 3bb5af6a06..36c1a41f67 100644 --- a/python/sglang/srt/entrypoints/openai/protocol.py +++ b/python/sglang/srt/entrypoints/openai/protocol.py @@ -407,6 +407,9 @@ class CompletionRequest(PDRoutingFields): ) images_config: Optional[Dict] = None + # Per-request processor overrides; only MiniCPM-V 4.7 reads them (downsample_mode, + # max_slice_nums), flattening images_/videos_kwargs into one request-wide profile. + mm_processor_kwargs: Optional[Dict] = None # Deprecated: use routed_dp_rank instead data_parallel_rank: Optional[int] = None @@ -966,6 +969,9 @@ class ChatCompletionRequest(PDRoutingFields): images_config: Optional[Dict] = None video_config: Optional[Dict] = None + # Per-request processor overrides; only MiniCPM-V 4.7 reads them (downsample_mode, + # max_slice_nums), flattening images_/videos_kwargs into one request-wide profile. + mm_processor_kwargs: Optional[Dict] = None # Custom logit processor for advanced sampling control custom_logit_processor: Optional[Union[List[Optional[str]], str]] = None diff --git a/python/sglang/srt/entrypoints/openai/serving_chat.py b/python/sglang/srt/entrypoints/openai/serving_chat.py index 4b361f28f7..ec56b6f709 100644 --- a/python/sglang/srt/entrypoints/openai/serving_chat.py +++ b/python/sglang/srt/entrypoints/openai/serving_chat.py @@ -1328,6 +1328,7 @@ class OpenAIServingChat(OpenAIServingBase): custom_logit_processor=request.custom_logit_processor, images_config=getattr(request, "images_config", None), video_config=_build_video_config(request), + mm_processor_kwargs=request.mm_processor_kwargs, image_max_dynamic_patch=img_max_dynamic_patch, video_max_dynamic_patch=vid_max_dynamic_patch, max_dynamic_patch=getattr(request, "max_dynamic_patch", None), diff --git a/python/sglang/srt/entrypoints/openai/serving_completions.py b/python/sglang/srt/entrypoints/openai/serving_completions.py index c2e903e8e5..e381ddc396 100644 --- a/python/sglang/srt/entrypoints/openai/serving_completions.py +++ b/python/sglang/srt/entrypoints/openai/serving_completions.py @@ -140,6 +140,7 @@ class OpenAIServingCompletion(OpenAIServingBase): custom_labels=custom_labels, custom_logit_processor=request.custom_logit_processor, images_config=getattr(request, "images_config", None), + mm_processor_kwargs=request.mm_processor_kwargs, ) return adapted_request, request diff --git a/python/sglang/srt/managers/io_struct.py b/python/sglang/srt/managers/io_struct.py index 45d6228dab..e645e0230f 100644 --- a/python/sglang/srt/managers/io_struct.py +++ b/python/sglang/srt/managers/io_struct.py @@ -355,6 +355,7 @@ class GenerateReqInput: # For Unlimited-OCR images_config: Optional[dict] = None + mm_processor_kwargs: Optional[dict] = None # Pre-computed delimiter indices for multi-item scoring. # Batch-level: List[List[int]] (one per request). After __getitem__: List[int]. @@ -1009,6 +1010,7 @@ class GenerateReqInput: if self.custom_logit_processor is not None else None ), + mm_processor_kwargs=self.mm_processor_kwargs, positional_embed_overrides=self._get_positional_embed_overrides_item(i), # If `__getitem__` is called, these bootstrap fields must be lists. bootstrap_host=( diff --git a/python/sglang/srt/models/dflash.py b/python/sglang/srt/models/dflash.py index 62f823be4f..03b3837779 100644 --- a/python/sglang/srt/models/dflash.py +++ b/python/sglang/srt/models/dflash.py @@ -189,6 +189,11 @@ def _get_dflash_layer_attention_params( ) +_ROPE_DIM_SUPERSEDED_KEYS = frozenset( + {"rope_dim", "partial_rotary_factor", "mrope_section", "mrope_interleaved"} +) + + class DFlashAttention(nn.Module): def __init__( self, config, layer_id: int, quant_config=None, prefix: str = "" @@ -242,6 +247,18 @@ class DFlashAttention(nn.Module): self.k_norm = RMSNorm(head_dim, eps=rms_norm_eps) rope_theta, rope_scaling = get_rope_config(config) + rotary_dim = head_dim + partial_rotary_factor = float(getattr(config, "partial_rotary_factor", 1.0)) + if rope_scaling is not None and "rope_dim" in rope_scaling: + # An explicit rope_dim is the trained rotary width (HF Qwen3 ignores the + # copied target partial/M-RoPE fields); drafts only see 1-D positions. + rotary_dim = int(rope_scaling["rope_dim"]) + partial_rotary_factor = 1.0 + rope_scaling = { + k: v + for k, v in rope_scaling.items() + if k not in _ROPE_DIM_SUPERSEDED_KEYS + } rope_is_neox_style = bool( getattr( config, "rope_is_neox_style", getattr(config, "is_neox_style", True) @@ -250,12 +267,12 @@ class DFlashAttention(nn.Module): max_position_embeddings = int(getattr(config, "max_position_embeddings", 32768)) self.rotary_emb = get_rope( head_dim, - rotary_dim=head_dim, + rotary_dim=rotary_dim, max_position=max_position_embeddings, base=rope_theta, rope_scaling=rope_scaling, is_neox_style=rope_is_neox_style, - partial_rotary_factor=float(getattr(config, "partial_rotary_factor", 1.0)), + partial_rotary_factor=partial_rotary_factor, ) self.scaling = head_dim**-0.5 diff --git a/python/sglang/srt/models/minicpmv.py b/python/sglang/srt/models/minicpmv.py index 4fa7b66a6e..87fa8bbaa5 100644 --- a/python/sglang/srt/models/minicpmv.py +++ b/python/sglang/srt/models/minicpmv.py @@ -21,10 +21,11 @@ # limitations under the License. """Inference-only MiniCPM-V model compatible with HuggingFace weights.""" +import re import types from array import array from functools import partial -from itertools import chain +from itertools import chain, groupby from typing import ( Any, Callable, @@ -45,10 +46,12 @@ from torch import nn from torch.nn.init import trunc_normal_ from transformers import PretrainedConfig +from sglang.srt.configs.qwen3_5 import Qwen3_5MoeTextConfig from sglang.srt.layers.linear import ReplicatedLinear from sglang.srt.layers.logits_processor import LogitsProcessor from sglang.srt.layers.quantization.base_config import QuantizationConfig from sglang.srt.managers.mm_utils import ( + MultiModalityDataPaddingPatternMultimodalTokens, MultiModalityDataPaddingPatternTokenPairs, general_mm_embed_routine, ) @@ -68,7 +71,8 @@ from sglang.srt.models.minicpmv_vit import ( ) from sglang.srt.models.qwen2 import Qwen2Config, Qwen2ForCausalLM from sglang.srt.models.qwen3 import Qwen3Config, Qwen3ForCausalLM -from sglang.srt.models.qwen3_5 import Qwen3_5ForCausalLM +from sglang.srt.models.qwen3_5 import Qwen3_5ForCausalLM, Qwen3_5MoeForCausalLM +from sglang.srt.models.utils import WeightsMapper from sglang.srt.utils import add_prefix, flatten_nested_list, get_device RawImageType = Union[Image.Image, torch.Tensor] @@ -571,10 +575,15 @@ class Resampler4_5(BaseResampler): return x +# 4.6+ ship their own ``model_type``; 4.7 checkpoints still carry a stale +# numeric ``version``, so ``model_type`` must win. +_MODEL_TYPE_VERSIONS = {"minicpmv4_6": (4, 6), "minicpmv4_7": (4, 7)} + + def get_version_by_config(config: PretrainedConfig) -> Tuple[int, ...]: - # 4.6 ships its own ``model_type`` instead of a numeric ``version``. - if getattr(config, "model_type", None) == "minicpmv4_6": - return 4, 6 + model_type = getattr(config, "model_type", None) + if model_type in _MODEL_TYPE_VERSIONS: + return _MODEL_TYPE_VERSIONS[model_type] version_float = getattr(config, "version", None) @@ -1397,7 +1406,8 @@ class MiniCPMV4_6(MiniCPMBaseModel): prefix: str = "", ): super().__init__(config=config, quant_config=quant_config, prefix=prefix) - assert self.version == (4, 6) + assert self.version in ((4, 6), (4, 7)) + self.capture_aux_hidden_states = False # ``Qwen3_5ForCausalLM`` returns plain hidden states (body only, no LM # head, no LogitsProcessor). Add them here so the downstream sampler # sees a ``LogitsProcessorOutput``. With ``tie_word_embeddings=True`` @@ -1422,9 +1432,13 @@ class MiniCPMV4_6(MiniCPMBaseModel): prefix: str = "", ) -> nn.Module: # 4.6 nests the LLM config under ``text_config``. - return Qwen3_5ForCausalLM( - config=config.text_config, quant_config=quant_config, prefix=prefix + text_config = config.text_config + llm_cls = ( + Qwen3_5MoeForCausalLM + if isinstance(text_config, Qwen3_5MoeTextConfig) + else Qwen3_5ForCausalLM ) + return llm_cls(config=text_config, quant_config=quant_config, prefix=prefix) def forward( self, @@ -1442,10 +1456,22 @@ class MiniCPMV4_6(MiniCPMBaseModel): forward_batch=forward_batch, **kwargs, ) + aux_hidden_states = None + if self.capture_aux_hidden_states: + hidden_states, aux_hidden_states = hidden_states return self.logits_processor( - input_ids, hidden_states, self.lm_head, forward_batch + input_ids, hidden_states, self.lm_head, forward_batch, aux_hidden_states ) + def set_dflash_layers_to_capture(self, layer_ids: List[int]): + if layer_ids is None: + raise ValueError( + "DFLASH requires explicit layer_ids for aux hidden capture." + ) + self.capture_aux_hidden_states = True + # Qwen3.5 layers capture their input, i.e. the previous layer's output. + self.llm.set_dflash_layers_to_capture([val + 1 for val in layer_ids]) + def init_vision_module( self, config: PretrainedConfig, @@ -1500,6 +1526,22 @@ class MiniCPMV4_6(MiniCPMBaseModel): result = torch.cat([item.feature for item in items]) return result.reshape(-1, result.shape[-1]) + # ``use_vit_merger`` is per request, and one batch can mix requests. + default = self.config.downsample_mode != "4x" + runs = groupby( + items, + key=lambda item: item.model_specific_data.get("use_vit_merger", default), + ) + return torch.cat( + [ + self._encode_patches(list(run), use_vit_merger) + for use_vit_merger, run in runs + ] + ) + + def _encode_patches( + self, items: List[MultimodalDataItem], use_vit_merger: bool + ) -> torch.Tensor: pixel_values = flatten_nested_list([item.feature for item in items]) tgt_sizes = torch.stack( flatten_nested_list([item.tgt_size for item in items]), dim=0 @@ -1530,8 +1572,6 @@ class MiniCPMV4_6(MiniCPMBaseModel): patch_attn_mask.size(2), device=patch_attn_mask.device ).unsqueeze(0) < mask_shapes.unsqueeze(1) - use_vit_merger = getattr(self.config, "downsample_mode", "16x") != "4x" - vision_embedding, tgt_sizes_out = self.vpm( all_pixel_values.type(dtype), patch_attention_mask=patch_attn_mask, @@ -1546,15 +1586,8 @@ class MiniCPMV4_6(MiniCPMBaseModel): get_video_feature = get_image_feature def pad_input_ids(self, input_ids: array, image_inputs: MultimodalInputs) -> array: - im_start_id: int = image_inputs.im_start_id - im_end_id: int = image_inputs.im_end_id - slice_start_id: int = image_inputs.slice_start_id - slice_end_id: int = image_inputs.slice_end_id - - media_token_pairs = [(im_start_id, im_end_id), (slice_start_id, slice_end_id)] - pattern = MultiModalityDataPaddingPatternTokenPairs( - media_token_pairs, data_start_token_ids=[im_start_id] - ) + # Every thumbnail / slice is its own item; pad each span with its own value. + pattern = MultiModalityDataPaddingPatternMultimodalTokens() return pattern.pad_input_tokens(input_ids, image_inputs) def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]): @@ -1606,11 +1639,65 @@ class MiniCPMV4_6(MiniCPMBaseModel): weight_loader(param, loaded_weight) +# 4.7 checkpoints keep top-level ``llm`` / ``vpm`` / ``resampler`` / ``vit_merger`` +# names and store connector MLPs as ``nn.Sequential``. +_V4_7_WEIGHT_PREFIXES = ( + ("llm.model.", "model.language_model."), + ("llm.lm_head.", "lm_head."), + ("vit_merger.", "model.vision_tower.vit_merger."), + ("vpm.", "model.vision_tower."), + ("resampler.", "model.merger."), +) + + +def _v4_7_to_v4_6_weight_name(name: str) -> str: + for old, new in _V4_7_WEIGHT_PREFIXES: + if name.startswith(old): + name = new + name[len(old) :] + break + name = re.sub(r"^(model\.merger\.mlp\.\d+)\.mlp\.0\.", r"\1.linear_1.", name) + return re.sub(r"^(model\.merger\.mlp\.\d+)\.mlp\.2\.", r"\1.linear_2.", name) + + +class MiniCPMV4_7(MiniCPMV4_6): + def __init__( + self, + config: PretrainedConfig, + quant_config: Optional[QuantizationConfig] = None, + prefix: str = "", + ): + super().__init__(config=config, quant_config=quant_config, prefix=prefix) + rope_parameters = config.text_config.rope_parameters or {} + self.is_mrope_enabled = "mrope_section" in rope_parameters + + def forward( + self, + input_ids: torch.Tensor, + positions: torch.Tensor, + forward_batch: ForwardBatch, + **kwargs: Any, + ) -> torch.Tensor: + if self.is_mrope_enabled: + positions = forward_batch.mrope_positions + return super().forward( + input_ids=input_ids, + positions=positions, + forward_batch=forward_batch, + **kwargs, + ) + + def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]): + return super().load_weights( + (_v4_7_to_v4_6_weight_name(name), weight) for name, weight in weights + ) + + _SUPPORT_VERSION = { (2, 6): MiniCPMV2_6, (4, 0): MiniCPMV4_0, (4, 5): MiniCPMV4_5, (4, 6): MiniCPMV4_6, + (4, 7): MiniCPMV4_7, } @@ -1632,8 +1719,8 @@ class MiniCPMV: @staticmethod def shared_experts_fusion_disable_reason(hf_config, quant_config): - # 4.6 nests a Qwen3.5 LLM under ``text_config``; every other version - # builds a dense LLM, for which the Qwen3.5 gate answers None. + # 4.6/4.7 nest a Qwen3.5 config (dense or MoE) under ``text_config``; + # older versions are dense, for which the Qwen3.5 gate answers None. return Qwen3_5ForCausalLM.shared_experts_fusion_disable_reason( getattr(hf_config, "text_config", hf_config), quant_config ) @@ -1646,11 +1733,12 @@ class MiniCPMV: ) -> None: super().__init__() - # 4.6 carries ``model_type == "minicpmv4_6"`` instead of a numeric + # 4.6+ carry their own ``model_type`` instead of a numeric # ``config.version``; older versionless configs keep the legacy # ``(2, 6)`` default. - if getattr(config, "model_type", None) == "minicpmv4_6": - version = (4, 6) + model_type = getattr(config, "model_type", None) + if model_type in _MODEL_TYPE_VERSIONS: + version = _MODEL_TYPE_VERSIONS[model_type] elif not hasattr(config, "version"): version = (2, 6) else: @@ -1748,7 +1836,30 @@ class MiniCPMV: # ``__name__`` — resolves the canonical 4.6 architecture name through # ``MiniCPMV``'s version-dispatch factory. class MiniCPMV4_6ForConditionalGeneration(MiniCPMV): + # Only for the quant config: its ``ignore`` list names modules in either the + # HF or the native 4.7 layout (see _V4_7_WEIGHT_PREFIXES), while + # load_weights remaps checkpoint names itself. + hf_to_sglang_mapper = WeightsMapper( + orig_to_new_substr={"self_attn.out_proj": "self_attn.proj"}, + orig_to_new_prefix={ + "model.language_model.": "llm.", + "model.vision_tower.": "vpm.", + "model.merger.": "resampler.", + "llm.model.": "llm.", + "llm.lm_head": "lm_head", + "vit_merger.": "vpm.vit_merger.", + }, + orig_to_new_suffix={".mlp.0": ".linear_1", ".mlp.2": ".linear_2"}, + ) + packed_modules_mapping = Qwen3_5ForCausalLM.packed_modules_mapping + + +class MiniCPMV4_7ForConditionalGeneration(MiniCPMV4_6ForConditionalGeneration): pass -EntryClass = [MiniCPMV, MiniCPMV4_6ForConditionalGeneration] +EntryClass = [ + MiniCPMV, + MiniCPMV4_6ForConditionalGeneration, + MiniCPMV4_7ForConditionalGeneration, +] diff --git a/python/sglang/srt/multimodal/processors/minicpmv4_6.py b/python/sglang/srt/multimodal/processors/minicpmv4_6.py index c2a0f67b1b..bf86f833c9 100644 --- a/python/sglang/srt/multimodal/processors/minicpmv4_6.py +++ b/python/sglang/srt/multimodal/processors/minicpmv4_6.py @@ -17,6 +17,7 @@ module collapses to a thin wrapper that delegates to it. from __future__ import annotations +import hashlib import math from itertools import chain from typing import Any, List, Optional, Sequence, Tuple, Union @@ -31,6 +32,10 @@ from sglang.srt.managers.schedule_batch import ( MultimodalProcessorOutput, ) from sglang.srt.models.minicpmv import MiniCPMV4_6ForConditionalGeneration +from sglang.srt.multimodal.cache import ( + CONTENT_HASH_PREFIX, + resolve_multimodal_item_hash, +) from sglang.srt.multimodal.processors.base_processor import ( BaseMultimodalProcessor, MultimodalSpecialTokens, @@ -127,16 +132,16 @@ def _reshape_by_patch(image: torch.Tensor, patch_size: int) -> torch.Tensor: return patches -def _flatten_patches( - per_item_pv: List[List[torch.Tensor]], - per_item_ts: List[List[List[int]]], -) -> Tuple[List[torch.Tensor], List[torch.Tensor]]: - """Per-item per-patch -> flat per-patch (source first, slices row-major).""" - flat_pv = list(chain.from_iterable(per_item_pv)) - flat_ts = [ - torch.tensor(ts, dtype=torch.int32) for ts in chain.from_iterable(per_item_ts) - ] - return flat_pv, flat_ts +def _layout_namespace( + downsample_mode: str, + tgt_size: List[int], + media_tgt_sizes: List[List[int]], + media_grids: List[List[int]], +) -> str: + # Pixels alone omit the layout: patches of different shape can pack to the + # same tensor while their embeddings and M-RoPE positions differ. + layout = f"{downsample_mode}|{tgt_size}|{media_tgt_sizes}|{media_grids}" + return CONTENT_HASH_PREFIX + hashlib.sha256(layout.encode()).hexdigest() class MiniCPMV4_6ImageProcessor: @@ -231,6 +236,14 @@ class MiniCPMV4_6ImageProcessor: min_error = error return best_grid + @property + def use_vit_merger(self) -> bool: + return self.downsample_mode != "4x" + + @property + def pad_divisor(self) -> int: + return 16 if self.use_vit_merger else 4 + def _normalize(self, t: torch.Tensor) -> torch.Tensor: t = t * self.rescale_factor return (t - self.image_mean.to(t.dtype)) / self.image_std.to(t.dtype) @@ -305,6 +318,7 @@ class MiniCPMV4_6MultimodalProcessor(BaseMultimodalProcessor): models = [MiniCPMV4_6ForConditionalGeneration] support_dynamic_frame_expansion = False gpu_image_decode = False + video_image_id = True def __init__(self, hf_config, server_args, _processor, *args, **kwargs): super().__init__(hf_config, server_args, _processor, *args, **kwargs) @@ -353,8 +367,6 @@ class MiniCPMV4_6MultimodalProcessor(BaseMultimodalProcessor): self.slice_start_id = self._token_id(self.slice_start_token) self.slice_end_id = self._token_id(self.slice_end_token) - self.pad_divisor = 16 if downsample_mode != "4x" else 4 - self.mm_tokens = MultimodalSpecialTokens( image_token=self.image_token, image_token_id=self.image_token_id, @@ -375,19 +387,20 @@ class MiniCPMV4_6MultimodalProcessor(BaseMultimodalProcessor): self, tgt_sizes: List[List[int]], grid: List[int], + pad_divisor: int, ) -> str: """``...`` (+ optional ``...`` rows) for one image or video frame; inner pads are ``_PAD_PLACEHOLDER`` (caller swaps back after splicing). """ h0, w0 = tgt_sizes[0] - n_src = (h0 * w0) // self.pad_divisor + n_src = (h0 * w0) // pad_divisor out = self.image_start_token + _PAD_PLACEHOLDER * n_src + self.image_end_token if len(tgt_sizes) > 1 and grid and grid[0] > 0 and grid[1] > 0: grid_y, grid_x = int(grid[0]), int(grid[1]) h_s, w_s = tgt_sizes[1] - n_slice = (h_s * w_s) // self.pad_divisor + n_slice = (h_s * w_s) // pad_divisor slice_chunk = ( self.slice_start_token + _PAD_PLACEHOLDER * n_slice @@ -401,9 +414,13 @@ class MiniCPMV4_6MultimodalProcessor(BaseMultimodalProcessor): self, index: int, frames: Sequence[Tuple[List[List[int]], List[int]]], + pad_divisor: int, + image_id: bool = True, ) -> str: """One image or one video. Image is a single-frame video.""" - body = "".join(self._expand_frame(ts, grid) for ts, grid in frames) + body = "".join(self._expand_frame(ts, grid, pad_divisor) for ts, grid in frames) + if not image_id: + return body return f"{self.image_id_start_token}{index}{self.image_id_end_token}" + body async def process_mm_data_async( @@ -433,19 +450,28 @@ class MiniCPMV4_6MultimodalProcessor(BaseMultimodalProcessor): images = base.images or [] videos = base.videos or [] + image_processor = self._image_processor_for(request_obj) + pad_divisor = image_processor.pad_divisor + # Image: one "frame" per image. Video: per-frame nesting kept so each # frame becomes its own ``...`` block in the expansion. - img_per_pv, img_per_ts, img_grids = self._preprocess_images(images) - vid_per_pv, vid_per_ts, vid_grids = self._preprocess_videos(videos) + img_per_pv, img_per_ts, img_grids = self._preprocess_images( + images, image_processor + ) + vid_per_pv, vid_per_ts, vid_grids = self._preprocess_videos( + videos, image_processor + ) prompt = self._splice_expansions( prompt, ( - self._expand_media(i, [(ts, gd)]) + self._expand_media(i, [(ts, gd)], pad_divisor) for i, (ts, gd) in enumerate(zip(img_per_ts, img_grids)) ), ( - self._expand_media(i, list(zip(fts, fgd))) + self._expand_media( + i, list(zip(fts, fgd)), pad_divisor, self.video_image_id + ) for i, (fts, fgd) in enumerate(zip(vid_per_ts, vid_grids)) ), ) @@ -462,20 +488,29 @@ class MiniCPMV4_6MultimodalProcessor(BaseMultimodalProcessor): self._build_items( input_ids_tensor, self.image_token_id, - _flatten_patches(img_per_pv, img_per_ts), + [ + (pvs, tss, [grid]) + for pvs, tss, grid in zip(img_per_pv, img_per_ts, img_grids) + ], Modality.IMAGE, + image_processor, ) ) - # Video: extra ``per-frame -> per-patch`` nesting; pre-flatten one - # level so ``_flatten_patches`` sees the same shape as image. - vid_pv_flat = [list(chain.from_iterable(v)) for v in vid_per_pv] - vid_ts_flat = [list(chain.from_iterable(v)) for v in vid_per_ts] + # Video: one media unit per video, its frames' patches flattened in order. mm_items.extend( self._build_items( input_ids_tensor, self.video_token_id, - _flatten_patches(vid_pv_flat, vid_ts_flat), + [ + ( + list(chain.from_iterable(pvs)), + list(chain.from_iterable(tss)), + grids, + ) + for pvs, tss, grids in zip(vid_per_pv, vid_per_ts, vid_grids) + ], Modality.VIDEO, + image_processor, ) ) @@ -483,24 +518,28 @@ class MiniCPMV4_6MultimodalProcessor(BaseMultimodalProcessor): mm_items=mm_items, input_ids=input_ids, im_token_id=self.image_token_id, + video_token_id=self.video_token_id, im_start_id=self.image_start_id, im_end_id=self.image_end_id, slice_start_id=self.slice_start_id, slice_end_id=self.slice_end_id, ) - def _preprocess_images(self, images): + def _image_processor_for(self, request_obj) -> MiniCPMV4_6ImageProcessor: + return self.image_processor + + def _preprocess_images(self, images, image_processor): if not images: return [], [], [] - out = self.image_processor.preprocess(images) + out = image_processor.preprocess(images) return out["pixel_values"], out["tgt_sizes"], out["grids"] - def _preprocess_videos(self, videos): + def _preprocess_videos(self, videos, image_processor): per_video_pv: List[List[List[torch.Tensor]]] = [] per_video_ts: List[List[List[List[int]]]] = [] per_video_grids: List[List[List[int]]] = [] for frames in videos: - out = self.image_processor.preprocess(list(frames)) + out = image_processor.preprocess(list(frames)) per_video_pv.append(out["pixel_values"]) per_video_ts.append(out["tgt_sizes"]) per_video_grids.append(out["grids"]) @@ -527,22 +566,33 @@ class MiniCPMV4_6MultimodalProcessor(BaseMultimodalProcessor): self, input_ids: torch.Tensor, pad_token_id: int, - flat: Tuple[List[torch.Tensor], List[torch.Tensor]], + media: List[Tuple[List[torch.Tensor], List[List[int]], List[List[int]]]], modality: Modality, + image_processor: MiniCPMV4_6ImageProcessor, ) -> List[MultimodalDataItem]: - flat_pv, flat_ts = flat + """``media``: per image / video, its patches' pixel values, tgt sizes + and slice grids (source patch first, slices row-major).""" + patches = [ + (pv, ts, _layout_namespace(image_processor.downsample_mode, ts, tss, grids)) + for pvs, tss, grids in media + for pv, ts in zip(pvs, tss) + ] runs = self.get_mm_items_offset(input_ids, pad_token_id) - if len(runs) != len(flat_pv): + if len(runs) != len(patches): raise RuntimeError( f"[minicpmv4_6] {modality} pad run / feature count mismatch: " - f"{len(runs)} runs vs {len(flat_pv)} patches" + f"{len(runs)} runs vs {len(patches)} patches" ) return [ MultimodalDataItem( feature=[pv], + hash=resolve_multimodal_item_hash(feature=[pv], namespace=namespace), offsets=[run], - model_specific_data={"tgt_size": [ts]}, + model_specific_data={ + "tgt_size": [torch.tensor(ts, dtype=torch.int32)], + "use_vit_merger": image_processor.use_vit_merger, + }, modality=modality, ) - for run, pv, ts in zip(runs, flat_pv, flat_ts) + for run, (pv, ts, namespace) in zip(runs, patches) ] diff --git a/python/sglang/srt/multimodal/processors/minicpmv4_7.py b/python/sglang/srt/multimodal/processors/minicpmv4_7.py new file mode 100644 index 0000000000..5b1e2e6b56 --- /dev/null +++ b/python/sglang/srt/multimodal/processors/minicpmv4_7.py @@ -0,0 +1,347 @@ +# Copyright 2026 The SGLang team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +"""SGLang multimodal processor for MiniCPM-V 4.7. + +The canvas M-RoPE layout ports the checkpoints' reference +``mrope_minicpmv4_7.py`` and must assign exactly the same positions. +""" + +from __future__ import annotations + +import logging +import math +from typing import List, Optional, Tuple + +import torch + +from sglang.srt.models.minicpmv import MiniCPMV4_7ForConditionalGeneration +from sglang.srt.multimodal.processors.minicpmv4_6 import ( + MiniCPMV4_6ImageProcessor, + MiniCPMV4_6MultimodalProcessor, +) + +logger = logging.getLogger(__name__) + +_SCOPED_KWARGS_KEYS = ("images_kwargs", "videos_kwargs", "audio_kwargs") +_STRUCTURAL_KEYS = ("im_start_id", "im_end_id", "slice_start_id", "slice_end_id") + +# (start, end, index into target_sizes) of one run of visual tokens. +_Span = Tuple[int, int, int] +_Frame = Tuple[_Span, List[_Span]] + + +def _parse_mm_processor_kwargs(raw) -> dict: + """Flatten HF-style ``images_kwargs`` / ``videos_kwargs``; top-level keys win.""" + if not isinstance(raw, dict): + return {} + flat = {} + for key in _SCOPED_KWARGS_KEYS: + if isinstance(raw.get(key), dict): + flat.update(raw[key]) + flat.update({k: v for k, v in raw.items() if k not in _SCOPED_KWARGS_KEYS}) + return flat + + +def _llm_grid(n_tokens: int, vit_h: int, vit_w: int) -> Tuple[int, int]: + """LLM-token grid for ``n_tokens`` merged from a ``vit_h x vit_w`` patch grid.""" + if n_tokens <= 0 or vit_h <= 0 or vit_w <= 0: + return 0, 0 + vit_total = vit_h * vit_w + if vit_total == n_tokens: + return vit_h, vit_w + if vit_total < n_tokens: + return 0, 0 + factor = int(round(math.sqrt(vit_total / n_tokens))) + if factor > 0 and vit_h % factor == 0 and vit_w % factor == 0: + height, width = vit_h // factor, vit_w // factor + if height * width == n_tokens: + return height, width + ratio = vit_h / vit_w + width = max(1, int(round(math.sqrt(n_tokens / ratio)))) + height = n_tokens // width + if height * width == n_tokens and height > 0: + return height, width + return 0, 0 + + +def build_image_bounds(input_ids: torch.Tensor, token_ids: dict) -> torch.Tensor: + """``[start, end)`` of every thumbnail / slice token run, shape (n, 2).""" + start_ids = [token_ids[k] for k in ("im_start_id", "slice_start_id")] + end_ids = [token_ids[k] for k in ("im_end_id", "slice_end_id")] + starts = torch.isin(input_ids, torch.tensor(start_ids)).nonzero().flatten() + 1 + ends = torch.isin(input_ids, torch.tensor(end_ids)).nonzero().flatten() + n = min(len(starts), len(ends)) + return torch.stack([starts[:n], ends[:n]], dim=-1) + + +def _frame_end(ids: List[int], frame: _Frame, structural: set, limit: int) -> int: + thumbnail, slices = frame + end = slices[-1][1] if slices else thumbnail[1] + while end < limit and ids[end] in structural: + end += 1 + return min(end, limit) + + +def _group_frames( + ids: List[int], bounds: List[List[int]], token_ids: dict, structural: set +) -> List[List[_Frame]]: + """Pair each thumbnail with its slices, then merge frames separated only by + structural / newline tokens into one video group.""" + frames: List[_Frame] = [] + for index, (start, end) in enumerate(bounds): + if frames and ids[start - 1] == token_ids["slice_start_id"]: + frames[-1][1].append((start, end, index)) + else: + frames.append(((start, end, index), [])) + + gap_ok = structural | {token_ids["newline_id"]} + groups = [[frames[0]]] + for prev, frame in zip(frames, frames[1:]): + gap = range(_frame_end(ids, prev, structural, len(ids)), frame[0][0] - 1) + if all(ids[p] in gap_ok for p in gap): + groups[-1].append(frame) + else: + groups.append([frame]) + return groups + + +def _fill_text(positions: torch.Tensor, start: int, end: int, pos: int) -> int: + if end <= start: + return pos + positions[:, start:end] = torch.arange(pos, pos + end - start) + return pos + end - start + + +def _fill_grid( + positions: torch.Tensor, + start: int, + base: int, + rows: torch.Tensor, + cols: torch.Tensor, +) -> None: + end = start + len(rows) * len(cols) + positions[0, start:end] = base + positions[1, start:end] = rows.repeat_interleave(len(cols)) + base + positions[2, start:end] = cols.repeat(len(rows)) + base + + +def _place_frame( + positions: torch.Tensor, + frame: _Frame, + frame_end: int, + base: int, + sizes: List[List[int]], +) -> int: + """Lay one thumbnail and its slice grid out on a shared canvas at ``base``; + returns the first position after the canvas.""" + (t_start, t_end, t_index), slices = frame + im_start = t_start - 1 + n_thumb = t_end - t_start + th, tw = _llm_grid(n_thumb, *sizes[t_index]) + + if slices: + cols = len(slices) + for k in range(len(slices) - 1): + if slices[k + 1][0] - slices[k][1] > 2: + cols = k + 1 + break + rows = len(slices) // cols + if rows * cols != len(slices): + rows, cols = 1, len(slices) + s0_start, s0_end, s0_index = slices[0] + sh, sw = _llm_grid(s0_end - s0_start, *sizes[s0_index]) + canvas_h, canvas_w = rows * sh, cols * sw + else: + canvas_h, canvas_w = max(th, 1), max(tw, 1) + + positions[:, im_start:frame_end] = base + positions[1:, im_start] = max(base - 1, 0) + if t_end < frame_end: + positions[1, t_end] = base + canvas_h + positions[2, t_end] = base + canvas_w + + if th > 0: + if slices and canvas_h > 0 and canvas_w > 0: + # The thumbnail is stretched over the full slice canvas. + t_rows = torch.linspace(0, canvas_h - 1, th).round().long() + t_cols = torch.linspace(0, canvas_w - 1, tw).round().long() + else: + t_rows, t_cols = torch.arange(th), torch.arange(tw) + _fill_grid(positions, t_start, base, t_rows, t_cols) + else: + _fill_text(positions, t_start, t_end, base) + + for k, (s_start, s_end, s_index) in enumerate(slices): + sh_k, sw_k = _llm_grid(s_end - s_start, *sizes[s_index]) + h_off, w_off = (k // cols) * sh, (k % cols) * sw + positions[1, s_start - 1] = base + h_off + positions[2, s_start - 1] = base + w_off + if s_end < frame_end: + positions[1, s_end] = base + h_off + max(sh_k - 1, 0) + positions[2, s_end] = base + w_off + max(sw_k - 1, 0) + if sh_k > 0: + rows_k = torch.arange(sh_k) + h_off + cols_k = torch.arange(sw_k) + w_off + _fill_grid(positions, s_start, base, rows_k, cols_k) + else: + _fill_text(positions, s_start, s_end, base) + + # Row-separating newlines sit one column past the canvas, on the row just ended. + for k in range(len(slices) - 1): + gap_start, gap_end = slices[k][1] + 1, slices[k + 1][0] - 1 + if gap_end > gap_start: + positions[1, gap_start:gap_end] = base + (k // cols + 1) * sh - 1 + positions[2, gap_start:gap_end] = base + cols * sw + + return base + max(canvas_h, canvas_w) + 1 + + +def compute_canvas_positions( + input_ids: torch.Tensor, + image_bounds: torch.Tensor, + target_sizes: List[List[int]], + token_ids: dict, +) -> torch.Tensor: + """(3, seq_len) ``[temporal, height, width]`` canvas M-RoPE positions.""" + ids = input_ids.tolist() + seq_len = len(ids) + positions = torch.arange(seq_len).expand(3, -1).clone() + if len(image_bounds) == 0: + return positions + + structural = {token_ids[k] for k in _STRUCTURAL_KEYS} + pos = cursor = 0 + for frames in _group_frames(ids, image_bounds.tolist(), token_ids, structural): + group_start = frames[0][0][0] - 1 + group_end = _frame_end(ids, frames[-1], structural, seq_len) + pos = _fill_text(positions, cursor, group_start, pos) + frame_cursor = group_start + for frame in frames: + im_start = frame[0][0] - 1 + if im_start > frame_cursor: + # The extra step keeps the next frame's halo off the gap. + pos = _fill_text(positions, frame_cursor, im_start, pos) + 1 + frame_end = _frame_end(ids, frame, structural, group_end) + pos = _place_frame(positions, frame, frame_end, pos, target_sizes) + frame_cursor = frame_end + cursor = group_end + _fill_text(positions, cursor, seq_len, pos) + return positions + + +def _sequential_positions(seq_len: int) -> torch.Tensor: + return torch.arange(seq_len).expand(3, -1) + + +def _rope_delta(positions: torch.Tensor) -> torch.Tensor: + """Decode offset: the first generated token sits one past the last prompt position.""" + delta = int(positions.amax()) + 1 - positions.shape[1] + return torch.tensor([[delta]], dtype=torch.int64) + + +class MiniCPMV4_7MultimodalProcessor(MiniCPMV4_6MultimodalProcessor): + models = [MiniCPMV4_7ForConditionalGeneration] + # 4.7 was trained without ```` in front of videos. + video_image_id = False + + def __init__(self, hf_config, server_args, _processor, *args, **kwargs): + super().__init__(hf_config, server_args, _processor, *args, **kwargs) + self.canvas_token_ids = self._resolve_canvas_token_ids(hf_config) + + def _image_processor_for(self, request_obj) -> MiniCPMV4_6ImageProcessor: + kwargs = _parse_mm_processor_kwargs( + getattr(request_obj, "mm_processor_kwargs", None) + ) + default = self.image_processor + + downsample_mode = kwargs.get("downsample_mode") or default.downsample_mode + if downsample_mode not in ("4x", "16x"): + logger.warning("Ignoring unknown downsample_mode %r", downsample_mode) + downsample_mode = default.downsample_mode + max_slice_nums = kwargs.get("max_slice_nums") + if max_slice_nums is None: + max_slice_nums = default.max_slice_nums + try: + max_slice_nums = int(max_slice_nums) + except (TypeError, ValueError): + logger.warning("Ignoring non-numeric max_slice_nums %r", max_slice_nums) + max_slice_nums = default.max_slice_nums + + if (downsample_mode, max_slice_nums) == ( + default.downsample_mode, + default.max_slice_nums, + ): + return default + return MiniCPMV4_6ImageProcessor( + max_slice_nums=max_slice_nums, + scale_resolution=default.scale_resolution, + patch_size=default.patch_size, + downsample_mode=downsample_mode, + ) + + def _resolve_canvas_token_ids(self, hf_config) -> Optional[dict]: + mrope_mode = str(getattr(hf_config, "mrope_mode", "")).strip().lower() + if ( + not getattr(hf_config, "uses_mrope_canvas", False) + and mrope_mode != "canvas" + ): + return None + + # Byte-level BPE vocabularies store "\n" as "\u010a", so encode it. + newline_ids = self.tokenizer.encode("\n", add_special_tokens=False) + from_tokenizer = { + "im_start_id": self.image_start_id, + "im_end_id": self.image_end_id, + "slice_start_id": self.slice_start_id, + "slice_end_id": self.slice_end_id, + "newline_id": newline_ids[0] if len(newline_ids) == 1 else None, + } + token_ids = { + "im_start_id": getattr(hf_config, "image_start_id", None), + "im_end_id": getattr(hf_config, "image_end_id", None), + "slice_start_id": getattr(hf_config, "slice_start_id", None), + "slice_end_id": getattr(hf_config, "slice_end_id", None), + "newline_id": getattr(hf_config, "newline_id", None), + } + token_ids = { + k: from_tokenizer[k] if v is None else v for k, v in token_ids.items() + } + if None in token_ids.values(): + logger.warning("Canvas M-RoPE disabled: token ids missing (%s)", token_ids) + return None + return {k: int(v) for k, v in token_ids.items()} + + def compute_mrope_positions(self, input_ids, mm_items): + ids = torch.tensor(list(input_ids), dtype=torch.long) + positions = self._canvas_positions(ids, mm_items) + # The Qwen3.5 backbone reads mrope_positions for every multimodal request. + if positions is None: + positions = _sequential_positions(len(ids)) + return positions, _rope_delta(positions) + + def _canvas_positions(self, ids, mm_items) -> Optional[torch.Tensor]: + if self.canvas_token_ids is None: + return None + # Items are appended images-first; the canvas walk needs them in prompt order. + ordered = sorted(mm_items, key=lambda item: item.offsets[0][0]) + target_sizes = [torch.as_tensor(item.tgt_size[0]).tolist() for item in ordered] + image_bounds = build_image_bounds(ids, self.canvas_token_ids) + if len(image_bounds) != len(target_sizes): + logger.warning( + "Canvas M-RoPE: %d image bounds vs %d patch grids; " + "using sequential positions", + len(image_bounds), + len(target_sizes), + ) + return None + return compute_canvas_positions( + ids, image_bounds, target_sizes, self.canvas_token_ids + ) + + +__all__ = ["MiniCPMV4_7MultimodalProcessor"] diff --git a/python/sglang/srt/parser/conversation.py b/python/sglang/srt/parser/conversation.py index 585ecdca55..dd6df9572e 100644 --- a/python/sglang/srt/parser/conversation.py +++ b/python/sglang/srt/parser/conversation.py @@ -1196,11 +1196,11 @@ def match_minicpm(model_path: str): # must NOT fall back to the legacy `minicpmv` conv template (which encodes # the old `(./)` placeholder used by 2.x/4.0/4.5). model_type = get_model_type(model_path) - if model_type == "minicpmv4_6": + if model_type in ("minicpmv4_6", "minicpmv4_7"): return None # For HF-hub paths (where config.json isn't on local disk yet), fall back # to a path-version check: exclude 4.6 and later, match only legacy 2.x/4.0/4.5. - if re.search(r"minicpm-(v|o)-4[._]6", model_path, re.IGNORECASE): + if re.search(r"minicpm-(v|o)-4[._][67]", model_path, re.IGNORECASE): return None match = re.search(r"minicpm-(v|o)", model_path, re.IGNORECASE) if match: diff --git a/python/sglang/srt/speculative/dspark_components/dspark_config.py b/python/sglang/srt/speculative/dspark_components/dspark_config.py index 50909f9f7a..48143d3fc7 100644 --- a/python/sglang/srt/speculative/dspark_components/dspark_config.py +++ b/python/sglang/srt/speculative/dspark_components/dspark_config.py @@ -193,8 +193,8 @@ def _get_text_config(config: Any) -> Any: return config -def _get_dspark_config(config: Any) -> dict: - cfg = _cfg_get(config, "dspark_config", None) +def _get_nested_config(config: Any, key: str) -> dict: + cfg = _cfg_get(config, key, None) if cfg is None: return {} if isinstance(cfg, dict): @@ -205,6 +205,14 @@ def _get_dspark_config(config: Any) -> dict: return {} +def _get_dspark_config(config: Any) -> dict: + # Published DSpark drafts nest their fields under dflash_config. + return { + **_get_nested_config(config, "dflash_config"), + **_get_nested_config(config, "dspark_config"), + } + + def parse_dspark_draft_config(*, draft_hf_config: Any) -> DSparkDraftConfig: base = parse_dflash_draft_config(draft_hf_config=draft_hf_config) diff --git a/python/sglang/srt/utils/hf_transformers/common.py b/python/sglang/srt/utils/hf_transformers/common.py index 5c90ddfd93..6d98a22bad 100644 --- a/python/sglang/srt/utils/hf_transformers/common.py +++ b/python/sglang/srt/utils/hf_transformers/common.py @@ -69,6 +69,7 @@ from sglang.srt.configs import ( MiniCPMHybridConfig, MiniCPMV4_6Config, MiniCPMV4_6VisionConfig, + MiniCPMV4_7Config, MiniMaxM3VLConfig, MultiModalityConfig, MuseGlimmerAssistantConfig, @@ -176,6 +177,7 @@ _CONFIG_REGISTRY: Dict[str, Type[PretrainedConfig]] = { MiniCPMHybridConfig, MiniCPMV4_6Config, MiniCPMV4_6VisionConfig, + MiniCPMV4_7Config, InklingModelConfig, InklingAudioConfig, InklingVisionConfig, diff --git a/test/registered/unit/api/v1/test_parity.py b/test/registered/unit/api/v1/test_parity.py index 4058fb4bb2..98bbc82c10 100644 --- a/test/registered/unit/api/v1/test_parity.py +++ b/test/registered/unit/api/v1/test_parity.py @@ -62,6 +62,7 @@ PYTHON_ONLY_REQUEST_FIELDS = frozenset( "max_thinking_tokens", "min_dynamic_patch", "mm_content_hashes", + "mm_processor_kwargs", "modalities", "multi_item_delimiter_indices", "need_wait_for_mm_inputs", diff --git a/test/registered/unit/multimodal/test_minicpmv4_7_canvas_mrope.py b/test/registered/unit/multimodal/test_minicpmv4_7_canvas_mrope.py new file mode 100644 index 0000000000..cc50d661e2 --- /dev/null +++ b/test/registered/unit/multimodal/test_minicpmv4_7_canvas_mrope.py @@ -0,0 +1,65 @@ +"""Canvas M-RoPE layout for MiniCPM-V 4.7 (CPU only). + +The expected positions come from the checkpoints' reference +``mrope_minicpmv4_7.py`` and pin the port to it. +""" + +import unittest + +import torch + +from sglang.srt.multimodal.processors.minicpmv4_7 import ( + build_image_bounds, + compute_canvas_positions, +) +from sglang.test.ci.ci_register import register_cpu_ci +from sglang.test.test_utils import CustomTestCase + +register_cpu_ci(est_time=5, suite="base-a-test-cpu") + +# Synthetic ids: the layout code only ever compares them for equality. +IM_START, IM_END, SLICE_START, SLICE_END, NEWLINE, PAD = 900, 901, 902, 903, 904, 905 +SPECIAL_IDS = { + "im_start_id": IM_START, + "im_end_id": IM_END, + "slice_start_id": SLICE_START, + "slice_end_id": SLICE_END, + "newline_id": NEWLINE, +} + + +def _canvas(input_ids, target_sizes): + ids = torch.tensor(input_ids, dtype=torch.long) + return compute_canvas_positions( + ids, build_image_bounds(ids, SPECIAL_IDS), target_sizes, SPECIAL_IDS + ) + + +class TestMiniCPMV4_7CanvasMRoPE(CustomTestCase): + def test_thumbnail_grid_places_rows_and_columns(self): + # 16 visual tokens on a 4x4 grid: halo sits one position below + # the base, one past the canvas. + positions = _canvas( + [10, 11, IM_START] + [PAD] * 16 + [IM_END, 12, 13], [(4, 4)] + ) + + self.assertEqual(positions[0].tolist(), [0, 1, 2] + [2] * 16 + [2, 7, 8]) + self.assertEqual( + positions[1].tolist(), + [0, 1, 1] + [2, 2, 2, 2, 3, 3, 3, 3, 4, 4, 4, 4, 5, 5, 5, 5] + [6, 7, 8], + ) + self.assertEqual( + positions[2].tolist(), + [0, 1, 1] + [2, 3, 4, 5, 2, 3, 4, 5, 2, 3, 4, 5, 2, 3, 4, 5] + [6, 7, 8], + ) + + def test_tightly_adjacent_frames_form_one_video_group(self): + frame = [IM_START] + [PAD] * 16 + [IM_END] + positions = _canvas([10, 11] + frame * 3 + [12, 13], [(4, 4)] * 3) + + # Each frame advances its own base instead of sharing the first one's. + self.assertTrue((positions[0][20:36] > positions[0][3:19].max()).all()) + + +if __name__ == "__main__": + unittest.main()