[Refactor] multimodal: Replace padding strategies with helpers (#44845)
Co-authored-by: Mick Qian <mickqian@users.noreply.github.com>
This commit is contained in:
@@ -251,6 +251,101 @@ def duplicate_pad_mm_input_ids(
|
||||
return padded_ids[:]
|
||||
|
||||
|
||||
def pad_mm_token_pairs(
|
||||
input_ids: array,
|
||||
mm_inputs: MultimodalInputs,
|
||||
*,
|
||||
data_token_pairs: Optional[List[Tuple[int, int]]],
|
||||
data_start_token_ids: Optional[List[int]] = None,
|
||||
) -> array:
|
||||
"""Pad enclosed tokens without mutating input_ids; record item start offsets.
|
||||
|
||||
data_start_token_ids identifies new items, so slices can reuse an item's pad value.
|
||||
By default, every start token begins a new item.
|
||||
"""
|
||||
if data_start_token_ids is None:
|
||||
data_start_token_ids = [s for s, _e in data_token_pairs]
|
||||
assert_int64_array(input_ids, "input_ids")
|
||||
pad_values = [item.pad_value for item in mm_inputs.mm_items]
|
||||
mm_inputs.data_offsets = []
|
||||
if data_token_pairs is None:
|
||||
data_token_pairs = [mm_inputs.im_start_id, mm_inputs.im_end_id]
|
||||
if data_token_pairs is None:
|
||||
print_warning_once(
|
||||
"No data_token_pairs provided, RadixAttention might be influenced."
|
||||
)
|
||||
return input_ids
|
||||
start_token_ids = {s for s, _e in data_token_pairs}
|
||||
end_tokens_ids = {e for _s, e in data_token_pairs}
|
||||
|
||||
padded_ids = array("q")
|
||||
last_idx = 0
|
||||
data_idx = -1
|
||||
|
||||
start_indices = [i for i, x in enumerate(input_ids) if x in start_token_ids]
|
||||
end_indices = [i for i, x in enumerate(input_ids) if x in end_tokens_ids]
|
||||
|
||||
if len(start_indices) != len(end_indices):
|
||||
return input_ids
|
||||
|
||||
for start_idx, end_idx in zip(start_indices, end_indices):
|
||||
padded_ids.extend(input_ids[last_idx : start_idx + 1])
|
||||
|
||||
if input_ids[start_idx] in data_start_token_ids:
|
||||
data_idx += 1
|
||||
mm_inputs.data_offsets += [start_idx]
|
||||
|
||||
if data_idx >= len(pad_values):
|
||||
data_idx = len(pad_values) - 1
|
||||
|
||||
num_tokens = end_idx - start_idx - 1
|
||||
pad_value = pad_values[data_idx]
|
||||
if num_tokens > 0:
|
||||
padded_ids.extend(array("q", [pad_value]) * num_tokens)
|
||||
|
||||
last_idx = end_idx
|
||||
|
||||
padded_ids.extend(input_ids[last_idx:])
|
||||
|
||||
assert len(input_ids) == len(padded_ids), "Length validation fails"
|
||||
return padded_ids
|
||||
|
||||
|
||||
def pad_mm_tokens(input_ids: array, mm_inputs: MultimodalInputs) -> array:
|
||||
"""Pad inclusive per-item offsets without mutating the input token array."""
|
||||
assert_int64_array(input_ids, "input_ids")
|
||||
if not input_ids or not mm_inputs.mm_items:
|
||||
return input_ids
|
||||
|
||||
# as_tensor(array) walks and boxes every token. View a bulk copy instead,
|
||||
# keeping the original request's unpadded token buffer unchanged.
|
||||
padded_input_ids = input_ids[:]
|
||||
input_ids_tensor = torch.frombuffer(padded_input_ids, dtype=torch.int64)
|
||||
|
||||
# Replace multimodal tokens using per-item offsets
|
||||
items_by_modality = defaultdict(list)
|
||||
for item in mm_inputs.mm_items:
|
||||
items_by_modality[item.modality].append(item)
|
||||
|
||||
token_id_map = {
|
||||
Modality.IMAGE: mm_inputs.im_token_id,
|
||||
Modality.AUDIO: mm_inputs.audio_token_id,
|
||||
Modality.VIDEO: mm_inputs.video_token_id,
|
||||
}
|
||||
|
||||
for modality, items in items_by_modality.items():
|
||||
token_id = token_id_map.get(modality)
|
||||
|
||||
if not items or token_id is None:
|
||||
continue
|
||||
|
||||
for i, item in enumerate(items):
|
||||
for offset in items[i].offsets:
|
||||
input_ids_tensor[offset[0] : offset[1] + 1] = item.pad_value
|
||||
|
||||
return padded_input_ids
|
||||
|
||||
|
||||
class MultiModalityDataPaddingPattern:
|
||||
"""
|
||||
Data tokens (like image tokens) often need special handling during padding
|
||||
@@ -273,6 +368,8 @@ class MultiModalityDataPaddingPatternTokenPairs(MultiModalityDataPaddingPattern)
|
||||
The padded value in a region enclosed by a token pair with be the same one, as the MultimodalDataItem's pad value
|
||||
|
||||
This strategy should be applied when data content is marked by start/end token pairs in the input sequence.
|
||||
|
||||
Kept for compatibility; new callers should use pad_mm_token_pairs.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
@@ -295,56 +392,21 @@ class MultiModalityDataPaddingPatternTokenPairs(MultiModalityDataPaddingPattern)
|
||||
"""
|
||||
This function will replace the data-tokens in between with pad_values accordingly
|
||||
"""
|
||||
assert_int64_array(input_ids, "input_ids")
|
||||
pad_values = [item.pad_value for item in mm_inputs.mm_items]
|
||||
data_token_pairs = self.data_token_id_pairs
|
||||
mm_inputs.data_offsets = []
|
||||
if data_token_pairs is None:
|
||||
data_token_pairs = [mm_inputs.im_start_id, mm_inputs.im_end_id]
|
||||
if data_token_pairs is None:
|
||||
print_warning_once(
|
||||
"No data_token_pairs provided, RadixAttention might be influenced."
|
||||
)
|
||||
return input_ids
|
||||
start_token_ids = {s for s, _e in data_token_pairs}
|
||||
end_tokens_ids = {e for _s, e in data_token_pairs}
|
||||
|
||||
padded_ids = array("q")
|
||||
last_idx = 0
|
||||
data_idx = -1
|
||||
|
||||
start_indices = [i for i, x in enumerate(input_ids) if x in start_token_ids]
|
||||
end_indices = [i for i, x in enumerate(input_ids) if x in end_tokens_ids]
|
||||
|
||||
if len(start_indices) != len(end_indices):
|
||||
return input_ids
|
||||
|
||||
for start_idx, end_idx in zip(start_indices, end_indices):
|
||||
padded_ids.extend(input_ids[last_idx : start_idx + 1])
|
||||
|
||||
if input_ids[start_idx] in self.data_start_token_ids:
|
||||
data_idx += 1
|
||||
mm_inputs.data_offsets += [start_idx]
|
||||
|
||||
if data_idx >= len(pad_values):
|
||||
data_idx = len(pad_values) - 1
|
||||
|
||||
num_tokens = end_idx - start_idx - 1
|
||||
pad_value = pad_values[data_idx]
|
||||
if num_tokens > 0:
|
||||
padded_ids.extend(array("q", [pad_value]) * num_tokens)
|
||||
|
||||
last_idx = end_idx
|
||||
|
||||
padded_ids.extend(input_ids[last_idx:])
|
||||
|
||||
assert len(input_ids) == len(padded_ids), "Length validation fails"
|
||||
return padded_ids
|
||||
data_start_token_ids = self.data_start_token_ids
|
||||
return pad_mm_token_pairs(
|
||||
input_ids,
|
||||
mm_inputs,
|
||||
data_token_pairs=data_token_pairs,
|
||||
data_start_token_ids=data_start_token_ids,
|
||||
)
|
||||
|
||||
|
||||
class MultiModalityDataPaddingPatternMultimodalTokens(MultiModalityDataPaddingPattern):
|
||||
"""In this pattern, data tokens should be represented as repetitions of a single token
|
||||
e.g. <image><image>....<image>, or <audio><audio>...<audio>
|
||||
|
||||
Kept for compatibility; new callers should use pad_mm_tokens.
|
||||
"""
|
||||
|
||||
def pad_input_tokens(self, input_ids: array, mm_inputs: MultimodalInputs) -> array:
|
||||
@@ -352,37 +414,7 @@ class MultiModalityDataPaddingPatternMultimodalTokens(MultiModalityDataPaddingPa
|
||||
Replaces multimodal tokens in input_ids with corresponding pad_values from mm_items.
|
||||
Each modality (image, audio, video) is handled separately based on its token_id.
|
||||
"""
|
||||
assert_int64_array(input_ids, "input_ids")
|
||||
if not input_ids or not mm_inputs.mm_items:
|
||||
return input_ids
|
||||
|
||||
# as_tensor(array) walks and boxes every token. View a bulk copy instead,
|
||||
# keeping the original request's unpadded token buffer unchanged.
|
||||
padded_input_ids = input_ids[:]
|
||||
input_ids_tensor = torch.frombuffer(padded_input_ids, dtype=torch.int64)
|
||||
|
||||
# Replace multimodal tokens using per-item offsets
|
||||
items_by_modality = defaultdict(list)
|
||||
for item in mm_inputs.mm_items:
|
||||
items_by_modality[item.modality].append(item)
|
||||
|
||||
token_id_map = {
|
||||
Modality.IMAGE: mm_inputs.im_token_id,
|
||||
Modality.AUDIO: mm_inputs.audio_token_id,
|
||||
Modality.VIDEO: mm_inputs.video_token_id,
|
||||
}
|
||||
|
||||
for modality, items in items_by_modality.items():
|
||||
token_id = token_id_map.get(modality)
|
||||
|
||||
if not items or token_id is None:
|
||||
continue
|
||||
|
||||
for i, item in enumerate(items):
|
||||
for offset in items[i].offsets:
|
||||
input_ids_tensor[offset[0] : offset[1] + 1] = item.pad_value
|
||||
|
||||
return padded_input_ids
|
||||
return pad_mm_tokens(input_ids, mm_inputs)
|
||||
|
||||
|
||||
# masked_scatter_ materializes the expanded [num_tokens, hidden] bool mask plus
|
||||
|
||||
@@ -25,8 +25,8 @@ from transformers import PretrainedConfig
|
||||
from sglang.srt.layers.quantization.base_config import QuantizationConfig
|
||||
from sglang.srt.layers.utils import PPMissingLayer
|
||||
from sglang.srt.managers.mm_utils import (
|
||||
MultiModalityDataPaddingPatternMultimodalTokens,
|
||||
general_mm_embed_routine,
|
||||
pad_mm_tokens,
|
||||
)
|
||||
from sglang.srt.managers.schedule_batch import (
|
||||
MultimodalDataItem,
|
||||
@@ -118,13 +118,12 @@ class BailingMMNativeForConditionalGeneration(nn.Module):
|
||||
self.linear_proj = PPMissingLayer()
|
||||
|
||||
self.is_mrope_enabled = "mrope_section" in config.llm_config.rope_parameters
|
||||
self.pattern = MultiModalityDataPaddingPatternMultimodalTokens()
|
||||
|
||||
def get_input_embeddings(self):
|
||||
return self.model.get_input_embeddings()
|
||||
|
||||
def pad_input_ids(self, input_ids: array, mm_inputs: MultimodalInputs) -> array:
|
||||
return self.pattern.pad_input_tokens(input_ids, mm_inputs)
|
||||
return pad_mm_tokens(input_ids, mm_inputs)
|
||||
|
||||
def _get_vision_feature(
|
||||
self, items: List[MultimodalDataItem], grid_thw: torch.Tensor
|
||||
|
||||
@@ -26,8 +26,8 @@ from sglang.srt.configs.bailing_hybrid import is_bailing_multi_gate_enabled
|
||||
from sglang.srt.layers.quantization.base_config import QuantizationConfig
|
||||
from sglang.srt.layers.utils import PPMissingLayer
|
||||
from sglang.srt.managers.mm_utils import (
|
||||
MultiModalityDataPaddingPatternMultimodalTokens,
|
||||
general_mm_embed_routine,
|
||||
pad_mm_tokens,
|
||||
)
|
||||
from sglang.srt.managers.schedule_batch import (
|
||||
MultimodalDataItem,
|
||||
@@ -124,13 +124,12 @@ class BailingMoeV3VLForConditionalGeneration(nn.Module):
|
||||
self.linear_proj = PPMissingLayer()
|
||||
|
||||
self.is_mrope_enabled = "mrope_section" in text_config.rope_parameters
|
||||
self.pattern = MultiModalityDataPaddingPatternMultimodalTokens()
|
||||
|
||||
def get_input_embeddings(self):
|
||||
return self.model.get_input_embeddings()
|
||||
|
||||
def pad_input_ids(self, input_ids: array, mm_inputs: MultimodalInputs) -> array:
|
||||
return self.pattern.pad_input_tokens(input_ids, mm_inputs)
|
||||
return pad_mm_tokens(input_ids, mm_inputs)
|
||||
|
||||
def _materialize_items(self, items: List[MultimodalDataItem]) -> torch.Tensor:
|
||||
return materialize_multimodal_features(
|
||||
|
||||
@@ -22,8 +22,8 @@ from sglang.srt.layers.linear import (
|
||||
from sglang.srt.layers.logits_processor import LogitsProcessorOutput
|
||||
from sglang.srt.layers.quantization.base_config import QuantizationConfig
|
||||
from sglang.srt.managers.mm_utils import (
|
||||
MultiModalityDataPaddingPatternMultimodalTokens,
|
||||
general_mm_embed_routine,
|
||||
pad_mm_tokens,
|
||||
)
|
||||
from sglang.srt.managers.schedule_batch import (
|
||||
Modality,
|
||||
@@ -153,8 +153,7 @@ class Cohere2VisionForConditionalGeneration(nn.Module):
|
||||
self.model = self.language_model.model
|
||||
|
||||
def pad_input_ids(self, input_ids: array, mm_inputs: MultimodalInputs) -> array:
|
||||
pattern = MultiModalityDataPaddingPatternMultimodalTokens()
|
||||
return pattern.pad_input_tokens(input_ids, mm_inputs)
|
||||
return pad_mm_tokens(input_ids, mm_inputs)
|
||||
|
||||
def get_image_feature(self, mm_input: List[MultimodalDataItem]) -> torch.Tensor:
|
||||
pixel_values = torch.cat(
|
||||
|
||||
@@ -35,8 +35,8 @@ from sglang.srt.layers.linear import ColumnParallelLinear, RowParallelLinear
|
||||
from sglang.srt.layers.logits_processor import LogitsProcessorOutput
|
||||
from sglang.srt.layers.quantization.base_config import QuantizationConfig
|
||||
from sglang.srt.managers.mm_utils import (
|
||||
MultiModalityDataPaddingPatternMultimodalTokens,
|
||||
embed_mm_inputs,
|
||||
pad_mm_tokens,
|
||||
)
|
||||
from sglang.srt.managers.schedule_batch import (
|
||||
MultimodalDataItem,
|
||||
@@ -214,8 +214,7 @@ class Cosmos3EdgeForConditionalGeneration(ArceeForCausalLM):
|
||||
)
|
||||
|
||||
def pad_input_ids(self, input_ids: array, mm_inputs: MultimodalInputs) -> array:
|
||||
pattern = MultiModalityDataPaddingPatternMultimodalTokens()
|
||||
return pattern.pad_input_tokens(input_ids, mm_inputs)
|
||||
return pad_mm_tokens(input_ids, mm_inputs)
|
||||
|
||||
@staticmethod
|
||||
def _as_tensor(value) -> Optional[torch.Tensor]:
|
||||
|
||||
@@ -49,8 +49,8 @@ from sglang.srt.layers.attention.vision import VisionAttention
|
||||
from sglang.srt.layers.logits_processor import LogitsProcessor
|
||||
from sglang.srt.layers.quantization import QuantizationConfig
|
||||
from sglang.srt.managers.mm_utils import (
|
||||
MultiModalityDataPaddingPatternTokenPairs,
|
||||
general_mm_embed_routine,
|
||||
pad_mm_token_pairs,
|
||||
)
|
||||
from sglang.srt.managers.schedule_batch import MultimodalDataItem, MultimodalInputs
|
||||
from sglang.srt.model_executor.forward_batch_info import ForwardBatch
|
||||
@@ -2001,9 +2001,9 @@ class MultiModalityCausalLM(MultiModalityPreTrainedModel):
|
||||
im_end_id = image_inputs.im_end_id
|
||||
media_token_pairs = [(im_start_id, im_end_id)]
|
||||
|
||||
helper = MultiModalityDataPaddingPatternTokenPairs(media_token_pairs)
|
||||
|
||||
return helper.pad_input_tokens(input_ids, image_inputs)
|
||||
return pad_mm_token_pairs(
|
||||
input_ids, image_inputs, data_token_pairs=media_token_pairs
|
||||
)
|
||||
|
||||
def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
|
||||
stacked_params_mapping = [
|
||||
|
||||
@@ -35,8 +35,8 @@ from transformers.models.vitdet.modeling_vitdet import get_rel_pos
|
||||
from sglang.srt.configs.deepseek_ocr import DeepseekVLV2Config, is_ocr2_config
|
||||
from sglang.srt.layers.quantization import QuantizationConfig
|
||||
from sglang.srt.managers.mm_utils import (
|
||||
MultiModalityDataPaddingPatternMultimodalTokens,
|
||||
general_mm_embed_routine,
|
||||
pad_mm_tokens,
|
||||
)
|
||||
from sglang.srt.managers.schedule_batch import MultimodalDataItem, MultimodalInputs
|
||||
from sglang.srt.model_executor.forward_batch_info import ForwardBatch
|
||||
@@ -1765,8 +1765,7 @@ class DeepseekOCRForCausalLM(nn.Module):
|
||||
return inputs_embeds
|
||||
|
||||
def pad_input_ids(self, input_ids: array, mm_inputs: MultimodalInputs) -> array:
|
||||
pattern = MultiModalityDataPaddingPatternMultimodalTokens()
|
||||
return pattern.pad_input_tokens(input_ids, mm_inputs)
|
||||
return pad_mm_tokens(input_ids, mm_inputs)
|
||||
|
||||
def get_image_feature(self, items: List[MultimodalDataItem]) -> torch.Tensor:
|
||||
vision_embeddings = self._process_image_input(items)
|
||||
|
||||
@@ -104,8 +104,8 @@ from sglang.srt.layers.rotary_embedding import get_rope_wrapper
|
||||
from sglang.srt.layers.utils import PPMissingLayer, get_layer_id
|
||||
from sglang.srt.layers.vocab_parallel_embedding import VocabParallelEmbedding
|
||||
from sglang.srt.managers.mm_utils import (
|
||||
MultiModalityDataPaddingPatternMultimodalTokens,
|
||||
embed_mm_inputs,
|
||||
pad_mm_tokens,
|
||||
)
|
||||
from sglang.srt.managers.schedule_batch import MM_PAD_SHIFT_VALUE, MultimodalInputs
|
||||
from sglang.srt.mem_cache.memory_pool import RadixAttention
|
||||
@@ -4456,9 +4456,7 @@ class DeepseekV4ForCausalLM(nn.Module):
|
||||
return self._routed_experts_weights_of_layer.value
|
||||
|
||||
def pad_input_ids(self, input_ids: array, mm_inputs) -> array:
|
||||
return MultiModalityDataPaddingPatternMultimodalTokens().pad_input_tokens(
|
||||
input_ids, mm_inputs
|
||||
)
|
||||
return pad_mm_tokens(input_ids, mm_inputs)
|
||||
|
||||
def get_image_feature(self, items):
|
||||
"""Return complete spans for the shared MM cache and chunk scheduler."""
|
||||
|
||||
@@ -13,8 +13,8 @@ from sglang.srt.configs.deepseekvl2 import (
|
||||
from sglang.srt.layers.linear import ReplicatedLinear
|
||||
from sglang.srt.layers.quantization.base_config import QuantizationConfig
|
||||
from sglang.srt.managers.mm_utils import (
|
||||
MultiModalityDataPaddingPatternMultimodalTokens,
|
||||
general_mm_embed_routine,
|
||||
pad_mm_tokens,
|
||||
)
|
||||
from sglang.srt.managers.schedule_batch import MultimodalDataItem, MultimodalInputs
|
||||
from sglang.srt.model_executor.forward_batch_info import ForwardBatch
|
||||
@@ -264,8 +264,7 @@ class DeepseekVL2ForCausalLM(nn.Module):
|
||||
weights_loader(param, loaded_weight)
|
||||
|
||||
def pad_input_ids(self, input_ids: array, mm_inputs: MultimodalInputs) -> array:
|
||||
pattern = MultiModalityDataPaddingPatternMultimodalTokens()
|
||||
return pattern.pad_input_tokens(input_ids, mm_inputs)
|
||||
return pad_mm_tokens(input_ids, mm_inputs)
|
||||
|
||||
def get_image_feature(self, items: List[MultimodalDataItem]):
|
||||
|
||||
|
||||
@@ -94,8 +94,8 @@ from sglang.srt.layers.vocab_parallel_embedding import (
|
||||
)
|
||||
from sglang.srt.lora.layers import unwrap_lora_layer
|
||||
from sglang.srt.managers.mm_utils import (
|
||||
MultiModalityDataPaddingPatternTokenPairs,
|
||||
general_mm_embed_routine,
|
||||
pad_mm_token_pairs,
|
||||
)
|
||||
from sglang.srt.managers.schedule_batch import (
|
||||
Modality,
|
||||
@@ -1667,9 +1667,9 @@ class Dots3LanguageModelForCausalLM(nn.Module):
|
||||
token_pairs.append((mm_inputs.im_start_id, mm_inputs.im_end_id))
|
||||
if mm_inputs.audio_start_id is not None and mm_inputs.audio_end_id is not None:
|
||||
token_pairs.append((mm_inputs.audio_start_id, mm_inputs.audio_end_id))
|
||||
return MultiModalityDataPaddingPatternTokenPairs(
|
||||
data_token_pairs=token_pairs or None
|
||||
).pad_input_tokens(input_ids, mm_inputs)
|
||||
return pad_mm_token_pairs(
|
||||
input_ids, mm_inputs, data_token_pairs=token_pairs or None
|
||||
)
|
||||
|
||||
def _get_precomputed_feature(self, items: List[MultimodalDataItem]) -> torch.Tensor:
|
||||
embed_param = next(self.model.embed_tokens.parameters())
|
||||
|
||||
@@ -13,8 +13,8 @@ from sglang.srt.layers.logits_processor import LogitsProcessor
|
||||
from sglang.srt.layers.quantization.base_config import QuantizationConfig
|
||||
from sglang.srt.layers.vocab_parallel_embedding import ParallelLMHead
|
||||
from sglang.srt.managers.mm_utils import (
|
||||
MultiModalityDataPaddingPatternMultimodalTokens,
|
||||
general_mm_embed_routine,
|
||||
pad_mm_tokens,
|
||||
)
|
||||
from sglang.srt.managers.schedule_batch import MultimodalDataItem, MultimodalInputs
|
||||
from sglang.srt.model_executor.forward_batch_info import ForwardBatch
|
||||
@@ -58,8 +58,7 @@ class DotsOCRForCausalLM(nn.Module):
|
||||
self.logits_processor = LogitsProcessor(config)
|
||||
|
||||
def pad_input_ids(self, input_ids: array, mm_inputs: MultimodalInputs) -> array:
|
||||
pattern = MultiModalityDataPaddingPatternMultimodalTokens()
|
||||
return pattern.pad_input_tokens(input_ids, mm_inputs)
|
||||
return pad_mm_tokens(input_ids, mm_inputs)
|
||||
|
||||
def get_image_feature(self, items: List[MultimodalDataItem]) -> torch.Tensor:
|
||||
# Extract pixel values and grid information (following reference pattern)
|
||||
|
||||
@@ -26,8 +26,8 @@ from torch import nn
|
||||
from sglang.srt.configs.dots_vlm import DotsVLMConfig
|
||||
from sglang.srt.layers.quantization.base_config import QuantizationConfig
|
||||
from sglang.srt.managers.mm_utils import (
|
||||
MultiModalityDataPaddingPatternMultimodalTokens,
|
||||
general_mm_embed_routine,
|
||||
pad_mm_tokens,
|
||||
)
|
||||
from sglang.srt.managers.schedule_batch import MultimodalDataItem, MultimodalInputs
|
||||
from sglang.srt.model_executor.forward_batch_info import ForwardBatch, PPProxyTensors
|
||||
@@ -136,8 +136,7 @@ class DotsVLMForCausalLM(nn.Module):
|
||||
def pad_input_ids(self, input_ids: array, mm_inputs: MultimodalInputs) -> array:
|
||||
"""Pad input_ids with multimodal tokens"""
|
||||
# Get image token ID for padding pattern
|
||||
pattern = MultiModalityDataPaddingPatternMultimodalTokens()
|
||||
padded_input_ids = pattern.pad_input_tokens(input_ids, mm_inputs)
|
||||
padded_input_ids = pad_mm_tokens(input_ids, mm_inputs)
|
||||
return padded_input_ids
|
||||
|
||||
def get_image_feature(self, items: List[MultimodalDataItem]) -> torch.Tensor:
|
||||
|
||||
@@ -42,8 +42,8 @@ from sglang.srt.layers.radix_attention import AttentionType, RadixAttention
|
||||
from sglang.srt.layers.rotary_embedding import get_rope
|
||||
from sglang.srt.layers.vocab_parallel_embedding import VocabParallelEmbedding
|
||||
from sglang.srt.managers.mm_utils import (
|
||||
MultiModalityDataPaddingPatternMultimodalTokens,
|
||||
general_mm_embed_routine,
|
||||
pad_mm_tokens,
|
||||
)
|
||||
from sglang.srt.managers.schedule_batch import (
|
||||
ForwardBatch,
|
||||
@@ -648,8 +648,7 @@ class EmbeddingGemma2Model(nn.Module):
|
||||
def pad_input_ids(
|
||||
self, input_ids: list[int], mm_inputs: MultimodalInputs
|
||||
) -> list[int]:
|
||||
pattern = MultiModalityDataPaddingPatternMultimodalTokens()
|
||||
return pattern.pad_input_tokens(input_ids, mm_inputs)
|
||||
return pad_mm_tokens(input_ids, mm_inputs)
|
||||
|
||||
def get_image_feature(self, items: list[MultimodalDataItem]) -> torch.Tensor:
|
||||
if self.vision_tower is None or self.embed_vision is None:
|
||||
|
||||
@@ -38,8 +38,8 @@ from sglang.srt.layers.quantization.base_config import QuantizationConfig
|
||||
from sglang.srt.layers.rotary_embedding import get_rope
|
||||
from sglang.srt.layers.vocab_parallel_embedding import ParallelLMHead
|
||||
from sglang.srt.managers.mm_utils import (
|
||||
MultiModalityDataPaddingPatternMultimodalTokens,
|
||||
general_mm_embed_routine,
|
||||
pad_mm_tokens,
|
||||
)
|
||||
from sglang.srt.managers.schedule_batch import MultimodalDataItem, MultimodalInputs
|
||||
from sglang.srt.model_executor.forward_batch_info import ForwardBatch
|
||||
@@ -590,8 +590,7 @@ class Ernie4_5_VLMoeForConditionalGeneration(nn.Module):
|
||||
self._visual_token_ids_tensor_cache = None
|
||||
|
||||
def pad_input_ids(self, input_ids: array, mm_inputs: MultimodalInputs) -> array:
|
||||
pattern = MultiModalityDataPaddingPatternMultimodalTokens()
|
||||
return pattern.pad_input_tokens(input_ids, mm_inputs)
|
||||
return pad_mm_tokens(input_ids, mm_inputs)
|
||||
|
||||
def _vision_forward(
|
||||
self,
|
||||
|
||||
@@ -32,8 +32,8 @@ from sglang.srt.layers.layernorm import Gemma3RMSNorm
|
||||
from sglang.srt.layers.logits_processor import LogitsProcessor
|
||||
from sglang.srt.layers.quantization.base_config import QuantizationConfig
|
||||
from sglang.srt.managers.mm_utils import (
|
||||
MultiModalityDataPaddingPatternTokenPairs,
|
||||
general_mm_embed_routine,
|
||||
pad_mm_token_pairs,
|
||||
)
|
||||
from sglang.srt.managers.schedule_batch import (
|
||||
MultimodalDataItem,
|
||||
@@ -209,8 +209,9 @@ class Gemma3ForConditionalGeneration(PreTrainedModel):
|
||||
im_end_id: int = image_inputs.im_end_id
|
||||
|
||||
media_token_pairs = [(im_start_id, im_end_id)]
|
||||
pattern = MultiModalityDataPaddingPatternTokenPairs(media_token_pairs)
|
||||
ids = pattern.pad_input_tokens(input_ids, image_inputs)
|
||||
ids = pad_mm_token_pairs(
|
||||
input_ids, image_inputs, data_token_pairs=media_token_pairs
|
||||
)
|
||||
return ids
|
||||
|
||||
def prepare_attn_masks(
|
||||
|
||||
@@ -20,8 +20,8 @@ from sglang.srt.layers.logits_processor import LogitsProcessor
|
||||
from sglang.srt.layers.quantization.base_config import QuantizationConfig
|
||||
from sglang.srt.layers.vocab_parallel_embedding import VocabParallelEmbedding
|
||||
from sglang.srt.managers.mm_utils import (
|
||||
MultiModalityDataPaddingPatternMultimodalTokens,
|
||||
general_mm_embed_routine,
|
||||
pad_mm_tokens,
|
||||
)
|
||||
from sglang.srt.managers.schedule_batch import (
|
||||
Modality,
|
||||
@@ -248,8 +248,7 @@ class Gemma3nForConditionalGeneration(PreTrainedModel):
|
||||
mm_inputs: MultimodalInputs,
|
||||
) -> array:
|
||||
"""Pad input IDs with image and audio tokens."""
|
||||
pattern = MultiModalityDataPaddingPatternMultimodalTokens()
|
||||
return pattern.pad_input_tokens(input_ids, mm_inputs)
|
||||
return pad_mm_tokens(input_ids, mm_inputs)
|
||||
|
||||
def get_input_embeddings(self) -> nn.Embedding:
|
||||
return self.language_model.get_input_embeddings()
|
||||
|
||||
@@ -41,8 +41,8 @@ from sglang.srt.layers.radix_attention import AttentionType, RadixAttention
|
||||
from sglang.srt.layers.rotary_embedding import get_rope
|
||||
from sglang.srt.layers.vocab_parallel_embedding import VocabParallelEmbedding
|
||||
from sglang.srt.managers.mm_utils import (
|
||||
MultiModalityDataPaddingPatternMultimodalTokens,
|
||||
general_mm_embed_routine,
|
||||
pad_mm_tokens,
|
||||
)
|
||||
from sglang.srt.managers.schedule_batch import (
|
||||
Modality,
|
||||
@@ -561,8 +561,7 @@ class DiffusionGemmaForBlockDiffusion(PreTrainedModel):
|
||||
def pad_input_ids(
|
||||
self, input_ids: List[int], mm_inputs: MultimodalInputs
|
||||
) -> List[int]:
|
||||
pattern = MultiModalityDataPaddingPatternMultimodalTokens()
|
||||
return pattern.pad_input_tokens(input_ids, mm_inputs)
|
||||
return pad_mm_tokens(input_ids, mm_inputs)
|
||||
|
||||
def get_image_feature(self, items: List[MultimodalDataItem]) -> torch.Tensor:
|
||||
vt = self.vision_tower
|
||||
|
||||
@@ -39,8 +39,8 @@ from sglang.srt.layers.quantization.base_config import QuantizationConfig
|
||||
from sglang.srt.layers.utils import PPMissingLayer
|
||||
from sglang.srt.layers.vocab_parallel_embedding import ParallelLMHead
|
||||
from sglang.srt.managers.mm_utils import (
|
||||
MultiModalityDataPaddingPatternMultimodalTokens,
|
||||
general_mm_embed_routine,
|
||||
pad_mm_tokens,
|
||||
)
|
||||
from sglang.srt.managers.schedule_batch import (
|
||||
Modality,
|
||||
@@ -301,8 +301,7 @@ class Gemma4ForConditionalGeneration(PreTrainedModel):
|
||||
mm_inputs: MultimodalInputs,
|
||||
) -> array:
|
||||
"""Pad input IDs with image and audio tokens."""
|
||||
pattern = MultiModalityDataPaddingPatternMultimodalTokens()
|
||||
return pattern.pad_input_tokens(input_ids, mm_inputs)
|
||||
return pad_mm_tokens(input_ids, mm_inputs)
|
||||
|
||||
def get_input_embeddings(self) -> nn.Embedding:
|
||||
return self.language_model.get_input_embeddings()
|
||||
|
||||
@@ -50,8 +50,8 @@ from sglang.srt.layers.rotary_embedding import get_rope
|
||||
from sglang.srt.layers.utils import PPMissingLayer
|
||||
from sglang.srt.layers.vocab_parallel_embedding import ParallelLMHead
|
||||
from sglang.srt.managers.mm_utils import (
|
||||
MultiModalityDataPaddingPatternMultimodalTokens,
|
||||
general_mm_embed_routine,
|
||||
pad_mm_tokens,
|
||||
)
|
||||
from sglang.srt.managers.schedule_batch import MultimodalDataItem, MultimodalInputs
|
||||
from sglang.srt.model_executor.forward_batch_info import ForwardBatch, PPProxyTensors
|
||||
@@ -612,8 +612,7 @@ class Glm4vForConditionalGeneration(nn.Module):
|
||||
self.capture_aux_hidden_states = False
|
||||
|
||||
def pad_input_ids(self, input_ids: array, mm_inputs: MultimodalInputs) -> array:
|
||||
pattern = MultiModalityDataPaddingPatternMultimodalTokens()
|
||||
return pattern.pad_input_tokens(input_ids, mm_inputs)
|
||||
return pad_mm_tokens(input_ids, mm_inputs)
|
||||
|
||||
def get_image_feature(self, items: List[MultimodalDataItem]) -> torch.Tensor:
|
||||
# in GLM-V, last dim is the same
|
||||
|
||||
@@ -67,8 +67,8 @@ from sglang.srt.layers.vocab_parallel_embedding import (
|
||||
get_embedding_tp_kwargs,
|
||||
)
|
||||
from sglang.srt.managers.mm_utils import (
|
||||
MultiModalityDataPaddingPatternMultimodalTokens,
|
||||
general_mm_embed_routine,
|
||||
pad_mm_tokens,
|
||||
prepare_mm_inputs,
|
||||
)
|
||||
from sglang.srt.managers.schedule_batch import MultimodalDataItem, MultimodalInputs
|
||||
@@ -1559,8 +1559,7 @@ class Glm5NextForConditionalGeneration(nn.Module):
|
||||
self.model.layers_to_capture = [val + 1 for val in layer_ids]
|
||||
|
||||
def pad_input_ids(self, input_ids: array, mm_inputs: MultimodalInputs) -> array:
|
||||
pattern = MultiModalityDataPaddingPatternMultimodalTokens()
|
||||
return pattern.pad_input_tokens(input_ids, mm_inputs)
|
||||
return pad_mm_tokens(input_ids, mm_inputs)
|
||||
|
||||
def get_image_feature(self, items: List[MultimodalDataItem]) -> torch.Tensor:
|
||||
pixel_values = torch.cat([item.feature for item in items], dim=0).type(
|
||||
|
||||
@@ -57,8 +57,8 @@ from sglang.srt.layers.vocab_parallel_embedding import (
|
||||
VocabParallelEmbedding,
|
||||
)
|
||||
from sglang.srt.managers.mm_utils import (
|
||||
MultiModalityDataPaddingPatternMultimodalTokens,
|
||||
general_mm_embed_routine,
|
||||
pad_mm_tokens,
|
||||
)
|
||||
from sglang.srt.managers.schedule_batch import MultimodalDataItem, MultimodalInputs
|
||||
from sglang.srt.model_executor.forward_batch_info import ForwardBatch
|
||||
@@ -1081,8 +1081,7 @@ class GlmImageForConditionalGeneration(nn.Module):
|
||||
self.logits_processor = LogitsProcessor(logits_config)
|
||||
|
||||
def pad_input_ids(self, input_ids: array, mm_inputs: MultimodalInputs) -> array:
|
||||
pattern = MultiModalityDataPaddingPatternMultimodalTokens()
|
||||
return pattern.pad_input_tokens(input_ids, mm_inputs)
|
||||
return pad_mm_tokens(input_ids, mm_inputs)
|
||||
|
||||
def get_image_feature(self, items: List[MultimodalDataItem]) -> torch.Tensor:
|
||||
"""Run vision encoder -> VQ-VAE encode -> embed_tokens on discrete indices."""
|
||||
|
||||
@@ -31,8 +31,8 @@ from transformers.models.glmasr.modeling_glmasr import (
|
||||
|
||||
from sglang.srt.layers.quantization.base_config import QuantizationConfig
|
||||
from sglang.srt.managers.mm_utils import (
|
||||
MultiModalityDataPaddingPatternMultimodalTokens,
|
||||
general_mm_embed_routine,
|
||||
pad_mm_tokens,
|
||||
)
|
||||
from sglang.srt.managers.schedule_batch import (
|
||||
Modality,
|
||||
@@ -87,10 +87,9 @@ class GlmAsrForConditionalGeneration(nn.Module):
|
||||
self.language_model = LlamaForCausalLM(
|
||||
config.text_config, quant_config, prefix=add_prefix("model", prefix)
|
||||
)
|
||||
self.pattern = MultiModalityDataPaddingPatternMultimodalTokens()
|
||||
|
||||
def pad_input_ids(self, input_ids: array, mm_inputs: MultimodalInputs) -> array:
|
||||
return self.pattern.pad_input_tokens(input_ids, mm_inputs)
|
||||
return pad_mm_tokens(input_ids, mm_inputs)
|
||||
|
||||
def get_audio_feature(self, items: List[MultimodalDataItem]) -> torch.Tensor:
|
||||
# The processor emits mel features as a batch of fixed-size 30s windows
|
||||
|
||||
@@ -52,8 +52,8 @@ from sglang.srt.layers.vocab_parallel_embedding import (
|
||||
VocabParallelEmbedding,
|
||||
)
|
||||
from sglang.srt.managers.mm_utils import (
|
||||
MultiModalityDataPaddingPatternMultimodalTokens,
|
||||
general_mm_embed_routine,
|
||||
pad_mm_tokens,
|
||||
)
|
||||
from sglang.srt.managers.schedule_batch import (
|
||||
Modality,
|
||||
@@ -998,7 +998,6 @@ class InklingForConditionalGeneration(nn.Module):
|
||||
if build_multimodal and self.config.vision_config.decoder_dmodel is not None
|
||||
else None
|
||||
)
|
||||
self.mm_pattern = MultiModalityDataPaddingPatternMultimodalTokens()
|
||||
|
||||
@property
|
||||
def model(self) -> nn.Module:
|
||||
@@ -1070,7 +1069,7 @@ class InklingForConditionalGeneration(nn.Module):
|
||||
# The processor expands one placeholder per media item into a run of the same
|
||||
# token id; the scheduler calls this to replace each run with the item's
|
||||
# pad_value (radix hash), which _embed_mm then masks on to scatter the embeds.
|
||||
return self.mm_pattern.pad_input_tokens(input_ids, mm_inputs)
|
||||
return pad_mm_tokens(input_ids, mm_inputs)
|
||||
|
||||
def get_input_embeddings(self) -> nn.Module:
|
||||
return self.llm.embed_tokens
|
||||
|
||||
@@ -9,8 +9,8 @@ from sglang.srt.layers.attention import vision_utils
|
||||
from sglang.srt.layers.moe.fused_moe_triton.layer import FusedMoE
|
||||
from sglang.srt.layers.quantization.base_config import QuantizationConfig
|
||||
from sglang.srt.managers.mm_utils import (
|
||||
MultiModalityDataPaddingPatternTokenPairs,
|
||||
general_mm_embed_routine,
|
||||
pad_mm_token_pairs,
|
||||
)
|
||||
from sglang.srt.managers.schedule_batch import (
|
||||
Modality,
|
||||
@@ -161,9 +161,10 @@ class InternS1ForConditionalGeneration(nn.Module):
|
||||
im_end_id: int = mm_inputs.im_end_id
|
||||
|
||||
media_token_pairs = [(im_start_id, im_end_id)]
|
||||
helper = MultiModalityDataPaddingPatternTokenPairs(media_token_pairs)
|
||||
|
||||
return helper.pad_input_tokens(input_ids, mm_inputs)
|
||||
return pad_mm_token_pairs(
|
||||
input_ids, mm_inputs, data_token_pairs=media_token_pairs
|
||||
)
|
||||
|
||||
def _mapping_interns1_name(self, name):
|
||||
names_map = {
|
||||
|
||||
@@ -27,8 +27,8 @@ from sglang.srt.layers.linear import ColumnParallelLinear, RowParallelLinear
|
||||
from sglang.srt.layers.moe.fused_moe_triton.layer import FusedMoE
|
||||
from sglang.srt.layers.quantization.base_config import QuantizationConfig
|
||||
from sglang.srt.managers.mm_utils import (
|
||||
MultiModalityDataPaddingPatternTokenPairs,
|
||||
general_mm_embed_routine,
|
||||
pad_mm_token_pairs,
|
||||
)
|
||||
from sglang.srt.managers.schedule_batch import (
|
||||
Modality,
|
||||
@@ -678,9 +678,10 @@ class InternVLChatModel(nn.Module):
|
||||
im_end_id: int = mm_inputs.im_end_id
|
||||
|
||||
media_token_pairs = [(im_start_id, im_end_id)]
|
||||
helper = MultiModalityDataPaddingPatternTokenPairs(media_token_pairs)
|
||||
|
||||
return helper.pad_input_tokens(input_ids, mm_inputs)
|
||||
return pad_mm_token_pairs(
|
||||
input_ids, mm_inputs, data_token_pairs=media_token_pairs
|
||||
)
|
||||
|
||||
def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
|
||||
expert_params_mapping = []
|
||||
|
||||
@@ -16,7 +16,7 @@ import sglang.srt.utils as utils
|
||||
from sglang.srt.configs.jet_vlm import JetVLMConfig
|
||||
from sglang.srt.layers.logits_processor import LogitsProcessorOutput
|
||||
from sglang.srt.layers.quantization.base_config import QuantizationConfig
|
||||
from sglang.srt.managers.mm_utils import MultiModalityDataPaddingPatternMultimodalTokens
|
||||
from sglang.srt.managers.mm_utils import pad_mm_tokens
|
||||
from sglang.srt.managers.schedule_batch import (
|
||||
Modality,
|
||||
MultimodalDataItem,
|
||||
@@ -135,8 +135,7 @@ class JetVLMForConditionalGeneration(nn.Module):
|
||||
weight_loader(param, loaded_weight)
|
||||
|
||||
def pad_input_ids(self, input_ids: array, mm_inputs: MultimodalInputs) -> array:
|
||||
pattern = MultiModalityDataPaddingPatternMultimodalTokens()
|
||||
return pattern.pad_input_tokens(input_ids, mm_inputs)
|
||||
return pad_mm_tokens(input_ids, mm_inputs)
|
||||
|
||||
|
||||
EntryClass = [JetVLMForConditionalGeneration]
|
||||
|
||||
@@ -28,8 +28,8 @@ from sglang.srt.layers.quantization.base_config import QuantizationConfig
|
||||
from sglang.srt.layers.quantization.modelslim.modelslim import ModelSlimConfig
|
||||
from sglang.srt.layers.quantization.quark.quark import QuarkConfig
|
||||
from sglang.srt.managers.mm_utils import (
|
||||
MultiModalityDataPaddingPatternMultimodalTokens,
|
||||
general_mm_embed_routine,
|
||||
pad_mm_tokens,
|
||||
)
|
||||
from sglang.srt.managers.schedule_batch import (
|
||||
Modality,
|
||||
@@ -769,8 +769,7 @@ class KimiK25ForConditionalGeneration(nn.Module):
|
||||
return mm_projection_auto(self.mm_projector, image_embeds)
|
||||
|
||||
def pad_input_ids(self, input_ids: array, mm_inputs: MultimodalInputs) -> array:
|
||||
pattern = MultiModalityDataPaddingPatternMultimodalTokens()
|
||||
return pattern.pad_input_tokens(input_ids, mm_inputs)
|
||||
return pad_mm_tokens(input_ids, mm_inputs)
|
||||
|
||||
@property
|
||||
def start_layer(self) -> int:
|
||||
|
||||
@@ -100,8 +100,8 @@ from sglang.srt.layers.vocab_parallel_embedding import (
|
||||
get_embedding_tp_kwargs,
|
||||
)
|
||||
from sglang.srt.managers.mm_utils import (
|
||||
MultiModalityDataPaddingPatternMultimodalTokens,
|
||||
general_mm_embed_routine,
|
||||
pad_mm_tokens,
|
||||
)
|
||||
from sglang.srt.managers.schedule_batch import (
|
||||
Modality,
|
||||
@@ -3761,8 +3761,7 @@ class KimiK3ForConditionalGeneration(nn.Module):
|
||||
return self.mm_projector(image_embeds)
|
||||
|
||||
def pad_input_ids(self, input_ids: array, mm_inputs: MultimodalInputs) -> array:
|
||||
pattern = MultiModalityDataPaddingPatternMultimodalTokens()
|
||||
return pattern.pad_input_tokens(input_ids, mm_inputs)
|
||||
return pad_mm_tokens(input_ids, mm_inputs)
|
||||
|
||||
@property
|
||||
def start_layer(self) -> int:
|
||||
|
||||
@@ -59,8 +59,8 @@ from sglang.srt.layers.activation import QuickGELU
|
||||
from sglang.srt.layers.moe.fused_moe_triton import FusedMoE
|
||||
from sglang.srt.layers.quantization.base_config import QuantizationConfig
|
||||
from sglang.srt.managers.mm_utils import (
|
||||
MultiModalityDataPaddingPatternMultimodalTokens,
|
||||
general_mm_embed_routine,
|
||||
pad_mm_tokens,
|
||||
)
|
||||
from sglang.srt.managers.schedule_batch import (
|
||||
Modality,
|
||||
@@ -194,8 +194,7 @@ class KimiVLForConditionalGeneration(nn.Module):
|
||||
return res
|
||||
|
||||
def pad_input_ids(self, input_ids: array, mm_inputs: MultimodalInputs) -> array:
|
||||
pattern = MultiModalityDataPaddingPatternMultimodalTokens()
|
||||
return pattern.pad_input_tokens(input_ids, mm_inputs)
|
||||
return pad_mm_tokens(input_ids, mm_inputs)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
|
||||
@@ -32,8 +32,8 @@ from sglang.srt.layers.linear import ColumnParallelLinear, RowParallelLinear
|
||||
from sglang.srt.layers.logits_processor import LogitsProcessor
|
||||
from sglang.srt.layers.quantization.base_config import QuantizationConfig
|
||||
from sglang.srt.managers.mm_utils import (
|
||||
MultiModalityDataPaddingPatternMultimodalTokens,
|
||||
general_mm_embed_routine,
|
||||
pad_mm_tokens,
|
||||
)
|
||||
from sglang.srt.managers.schedule_batch import (
|
||||
MultimodalDataItem,
|
||||
@@ -172,8 +172,7 @@ class Lfm2VlForConditionalGeneration(nn.Module):
|
||||
self.logits_processor = LogitsProcessor(config.text_config)
|
||||
|
||||
def pad_input_ids(self, input_ids: array, mm_inputs: MultimodalInputs) -> array:
|
||||
pattern = MultiModalityDataPaddingPatternMultimodalTokens()
|
||||
result = pattern.pad_input_tokens(input_ids, mm_inputs)
|
||||
result = pad_mm_tokens(input_ids, mm_inputs)
|
||||
return result
|
||||
|
||||
def get_input_embeddings(self) -> nn.Embedding:
|
||||
|
||||
@@ -40,8 +40,8 @@ import torch.nn as nn
|
||||
|
||||
from sglang.srt.layers.layernorm import RMSNorm
|
||||
from sglang.srt.managers.mm_utils import (
|
||||
MultiModalityDataPaddingPatternMultimodalTokens,
|
||||
general_mm_embed_routine,
|
||||
pad_mm_tokens,
|
||||
)
|
||||
from sglang.srt.managers.schedule_batch import MultimodalDataItem, MultimodalInputs
|
||||
from sglang.srt.model_executor.forward_batch_info import ForwardBatch
|
||||
@@ -127,8 +127,7 @@ class LightOnOCRForConditionalGeneration(nn.Module):
|
||||
)
|
||||
|
||||
def pad_input_ids(self, input_ids: array, mm_inputs: MultimodalInputs) -> array:
|
||||
pattern = MultiModalityDataPaddingPatternMultimodalTokens()
|
||||
return pattern.pad_input_tokens(input_ids, mm_inputs)
|
||||
return pad_mm_tokens(input_ids, mm_inputs)
|
||||
|
||||
def get_image_feature(self, items: List[MultimodalDataItem]) -> torch.Tensor:
|
||||
"""Process images through vision encoder and projection pipeline."""
|
||||
|
||||
@@ -29,8 +29,8 @@ from sglang.srt.configs.kimi_vl_moonvit import MoonViTConfig
|
||||
from sglang.srt.configs.locate_anything import LocateAnythingConfig
|
||||
from sglang.srt.layers.quantization.base_config import QuantizationConfig
|
||||
from sglang.srt.managers.mm_utils import (
|
||||
MultiModalityDataPaddingPatternMultimodalTokens,
|
||||
general_mm_embed_routine,
|
||||
pad_mm_tokens,
|
||||
)
|
||||
from sglang.srt.managers.schedule_batch import (
|
||||
Modality,
|
||||
@@ -134,8 +134,7 @@ class LocateAnythingForConditionalGeneration(nn.Module):
|
||||
return self.multi_modal_projector(torch.cat(image_features))
|
||||
|
||||
def pad_input_ids(self, input_ids: array, mm_inputs: MultimodalInputs) -> array:
|
||||
pattern = MultiModalityDataPaddingPatternMultimodalTokens()
|
||||
return pattern.pad_input_tokens(input_ids, mm_inputs)
|
||||
return pad_mm_tokens(input_ids, mm_inputs)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
|
||||
@@ -15,8 +15,8 @@ from sglang.srt.layers.conv import Conv2dLayer
|
||||
from sglang.srt.layers.linear import ColumnParallelLinear, RowParallelLinear
|
||||
from sglang.srt.layers.quantization.base_config import QuantizationConfig
|
||||
from sglang.srt.managers.mm_utils import (
|
||||
MultiModalityDataPaddingPatternMultimodalTokens,
|
||||
general_mm_embed_routine,
|
||||
pad_mm_tokens,
|
||||
)
|
||||
from sglang.srt.managers.schedule_batch import (
|
||||
Modality,
|
||||
@@ -505,8 +505,7 @@ class MiDashengLMModel(nn.Module):
|
||||
|
||||
def pad_input_ids(self, input_ids: array, mm_inputs: MultimodalInputs) -> array:
|
||||
"""Pad input IDs with multimodal tokens."""
|
||||
pattern = MultiModalityDataPaddingPatternMultimodalTokens()
|
||||
return pattern.pad_input_tokens(input_ids, mm_inputs)
|
||||
return pad_mm_tokens(input_ids, mm_inputs)
|
||||
|
||||
def get_audio_feature(self, items: List[MultimodalDataItem]) -> torch.Tensor:
|
||||
"""Process audio inputs and return embeddings.
|
||||
|
||||
@@ -86,8 +86,8 @@ from sglang.srt.layers.vocab_parallel_embedding import (
|
||||
)
|
||||
from sglang.srt.lora.layers import unwrap_lora_layer
|
||||
from sglang.srt.managers.mm_utils import (
|
||||
MultiModalityDataPaddingPatternMultimodalTokens,
|
||||
general_mm_embed_routine,
|
||||
pad_mm_tokens,
|
||||
)
|
||||
from sglang.srt.managers.schedule_batch import (
|
||||
Modality,
|
||||
@@ -1339,8 +1339,7 @@ class MiMoV2ForCausalLM(nn.Module, AudioEncoderMixin):
|
||||
return self.model.get_input_embedding(input_ids)
|
||||
|
||||
def pad_input_ids(self, input_ids: array, mm_inputs: MultimodalInputs) -> array:
|
||||
pattern = MultiModalityDataPaddingPatternMultimodalTokens()
|
||||
return pattern.pad_input_tokens(input_ids, mm_inputs)
|
||||
return pad_mm_tokens(input_ids, mm_inputs)
|
||||
|
||||
def preprocess_mm_for_encoder(self, mm_data, modality, config):
|
||||
if self._encoder_processor is None:
|
||||
|
||||
@@ -13,8 +13,8 @@ from typing import Any, Iterable, List, Optional, Tuple
|
||||
import torch
|
||||
|
||||
from sglang.srt.managers.mm_utils import (
|
||||
MultiModalityDataPaddingPatternMultimodalTokens,
|
||||
general_mm_embed_routine,
|
||||
pad_mm_tokens,
|
||||
)
|
||||
from sglang.srt.managers.schedule_batch import MultimodalInputs
|
||||
from sglang.srt.model_executor.forward_batch_info import ForwardBatch, PPProxyTensors
|
||||
@@ -58,8 +58,7 @@ class MiMoV2ASRForCausalLM(MiMoForCausalLM, AudioEncoderMixin):
|
||||
self.build_audio_encoder(MiMoAudioEncoderConfig(**config.audio_config))
|
||||
|
||||
def pad_input_ids(self, input_ids: array, mm_inputs: MultimodalInputs) -> array:
|
||||
pattern = MultiModalityDataPaddingPatternMultimodalTokens()
|
||||
return pattern.pad_input_tokens(input_ids, mm_inputs)
|
||||
return pad_mm_tokens(input_ids, mm_inputs)
|
||||
|
||||
def get_input_embeddings(self):
|
||||
if getattr(self.config, "encoder_only", False):
|
||||
|
||||
@@ -40,8 +40,8 @@ from transformers.models.whisper.modeling_whisper import (
|
||||
|
||||
from sglang.srt.layers.quantization import QuantizationConfig
|
||||
from sglang.srt.managers.mm_utils import (
|
||||
MultiModalityDataPaddingPatternTokenPairs,
|
||||
general_mm_embed_routine,
|
||||
pad_mm_token_pairs,
|
||||
)
|
||||
from sglang.srt.managers.schedule_batch import (
|
||||
MultimodalDataItem,
|
||||
@@ -1532,11 +1532,13 @@ class MiniCPMO(MiniCPMBaseModel):
|
||||
(mm_input.audio_start_id, mm_input.audio_end_id),
|
||||
]
|
||||
data_start_token_ids = [im_start_id, mm_input.audio_start_id]
|
||||
pattern = MultiModalityDataPaddingPatternTokenPairs(
|
||||
data_token_pairs=data_token_pairs, data_start_token_ids=data_start_token_ids
|
||||
)
|
||||
|
||||
return pattern.pad_input_tokens(input_ids, mm_input)
|
||||
return pad_mm_token_pairs(
|
||||
input_ids,
|
||||
mm_input,
|
||||
data_token_pairs=data_token_pairs,
|
||||
data_start_token_ids=data_start_token_ids,
|
||||
)
|
||||
|
||||
def _get_feat_extract_output_lengths(self, input_lengths: torch.LongTensor):
|
||||
"""
|
||||
|
||||
@@ -51,9 +51,9 @@ from sglang.srt.layers.linear import ReplicatedLinear
|
||||
from sglang.srt.layers.logits_processor import LogitsProcessor
|
||||
from sglang.srt.layers.quantization.base_config import QuantizationConfig
|
||||
from sglang.srt.managers.mm_utils import (
|
||||
MultiModalityDataPaddingPatternMultimodalTokens,
|
||||
MultiModalityDataPaddingPatternTokenPairs,
|
||||
general_mm_embed_routine,
|
||||
pad_mm_token_pairs,
|
||||
pad_mm_tokens,
|
||||
)
|
||||
from sglang.srt.managers.schedule_batch import (
|
||||
MultimodalDataItem,
|
||||
@@ -1003,12 +1003,13 @@ class MiniCPMV2_6(MiniCPMBaseModel):
|
||||
media_token_pairs = [(im_start_id, im_end_id), (slice_start_id, slice_end_id)]
|
||||
# Only increment data_idx on im_start (not slice_start) so all slices
|
||||
# within one image share the same pad_value for per-image caching.
|
||||
pattern = MultiModalityDataPaddingPatternTokenPairs(
|
||||
media_token_pairs, data_start_token_ids=[im_start_id]
|
||||
return pad_mm_token_pairs(
|
||||
input_ids,
|
||||
image_inputs,
|
||||
data_token_pairs=media_token_pairs,
|
||||
data_start_token_ids=[im_start_id],
|
||||
)
|
||||
|
||||
return pattern.pad_input_tokens(input_ids, image_inputs)
|
||||
|
||||
|
||||
class MiniCPMV4_0(MiniCPMBaseModel):
|
||||
packed_modules_mapping = {
|
||||
@@ -1169,12 +1170,13 @@ class MiniCPMV4_0(MiniCPMBaseModel):
|
||||
media_token_pairs = [(im_start_id, im_end_id), (slice_start_id, slice_end_id)]
|
||||
# Only increment data_idx on im_start (not slice_start) so all slices
|
||||
# within one image share the same pad_value for per-image caching.
|
||||
pattern = MultiModalityDataPaddingPatternTokenPairs(
|
||||
media_token_pairs, data_start_token_ids=[im_start_id]
|
||||
return pad_mm_token_pairs(
|
||||
input_ids,
|
||||
image_inputs,
|
||||
data_token_pairs=media_token_pairs,
|
||||
data_start_token_ids=[im_start_id],
|
||||
)
|
||||
|
||||
return pattern.pad_input_tokens(input_ids, image_inputs)
|
||||
|
||||
|
||||
class MiniCPMV4_5(MiniCPMBaseModel):
|
||||
packed_modules_mapping = {
|
||||
@@ -1339,12 +1341,13 @@ class MiniCPMV4_5(MiniCPMBaseModel):
|
||||
media_token_pairs = [(im_start_id, im_end_id), (slice_start_id, slice_end_id)]
|
||||
# Only increment data_idx on im_start (not slice_start) so all slices
|
||||
# within one image share the same pad_value for per-image caching.
|
||||
pattern = MultiModalityDataPaddingPatternTokenPairs(
|
||||
media_token_pairs, data_start_token_ids=[im_start_id]
|
||||
return pad_mm_token_pairs(
|
||||
input_ids,
|
||||
image_inputs,
|
||||
data_token_pairs=media_token_pairs,
|
||||
data_start_token_ids=[im_start_id],
|
||||
)
|
||||
|
||||
return pattern.pad_input_tokens(input_ids, image_inputs)
|
||||
|
||||
def eval(self):
|
||||
super().eval()
|
||||
return self
|
||||
@@ -1587,8 +1590,7 @@ class MiniCPMV4_6(MiniCPMBaseModel):
|
||||
|
||||
def pad_input_ids(self, input_ids: array, image_inputs: MultimodalInputs) -> array:
|
||||
# Every thumbnail / slice is its own item; pad each span with its own value.
|
||||
pattern = MultiModalityDataPaddingPatternMultimodalTokens()
|
||||
return pattern.pad_input_tokens(input_ids, image_inputs)
|
||||
return pad_mm_tokens(input_ids, image_inputs)
|
||||
|
||||
def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
|
||||
"""Remap 4.6 prefixes (``model.{vision_tower,merger,language_model}``)
|
||||
|
||||
@@ -17,8 +17,8 @@ from sglang.srt.layers.utils import PPMissingLayer
|
||||
from sglang.srt.layers.utils.common import get_layer_id
|
||||
from sglang.srt.layers.vocab_parallel_embedding import ParallelLMHead
|
||||
from sglang.srt.managers.mm_utils import (
|
||||
MultiModalityDataPaddingPatternMultimodalTokens,
|
||||
general_mm_embed_routine,
|
||||
pad_mm_tokens,
|
||||
)
|
||||
from sglang.srt.managers.schedule_batch import (
|
||||
MultimodalDataItem,
|
||||
@@ -197,9 +197,7 @@ class MiniMaxM3SparseForConditionalGeneration(nn.Module):
|
||||
)
|
||||
|
||||
def pad_input_ids(self, input_ids: array, mm_inputs: MultimodalInputs) -> array:
|
||||
return MultiModalityDataPaddingPatternMultimodalTokens().pad_input_tokens(
|
||||
input_ids, mm_inputs
|
||||
)
|
||||
return pad_mm_tokens(input_ids, mm_inputs)
|
||||
|
||||
def get_image_feature(self, items: List[MultimodalDataItem]) -> torch.Tensor:
|
||||
return get_image_feature(self.vision_tower, items, self.use_data_parallel)
|
||||
|
||||
@@ -24,8 +24,8 @@ from sglang.srt.layers.logits_processor import LogitsProcessor
|
||||
from sglang.srt.layers.moe.fused_moe_triton import FusedMoE
|
||||
from sglang.srt.layers.quantization import QuantizationConfig
|
||||
from sglang.srt.managers.mm_utils import (
|
||||
MultiModalityDataPaddingPatternMultimodalTokens,
|
||||
general_mm_embed_routine,
|
||||
pad_mm_tokens,
|
||||
)
|
||||
from sglang.srt.managers.schedule_batch import (
|
||||
Modality,
|
||||
@@ -502,7 +502,6 @@ class Llama4ForConditionalGeneration(nn.Module):
|
||||
self.logits_processor = LogitsProcessor(
|
||||
config.text_config if hasattr(config, "text_config") else config
|
||||
)
|
||||
self.padding_pattern = MultiModalityDataPaddingPatternMultimodalTokens()
|
||||
|
||||
def _has_vision_weights(self, config) -> bool:
|
||||
"""Check if the model has vision components by examining the checkpoint."""
|
||||
@@ -555,7 +554,7 @@ class Llama4ForConditionalGeneration(nn.Module):
|
||||
return False
|
||||
|
||||
def pad_input_ids(self, input_ids: array, mm_inputs: MultimodalInputs) -> array:
|
||||
return self.padding_pattern.pad_input_tokens(input_ids, mm_inputs)
|
||||
return pad_mm_tokens(input_ids, mm_inputs)
|
||||
|
||||
def get_image_feature(
|
||||
self,
|
||||
|
||||
@@ -48,8 +48,8 @@ from sglang.srt.layers.vocab_parallel_embedding import (
|
||||
VocabParallelEmbedding,
|
||||
)
|
||||
from sglang.srt.managers.mm_utils import (
|
||||
MultiModalityDataPaddingPatternMultimodalTokens,
|
||||
general_mm_embed_routine,
|
||||
pad_mm_tokens,
|
||||
)
|
||||
from sglang.srt.managers.schedule_batch import MultimodalDataItem, MultimodalInputs
|
||||
from sglang.srt.model_executor.forward_batch_info import ForwardBatch
|
||||
@@ -989,9 +989,7 @@ class MuseGlimmerForConditionalGeneration(MuseGlimmerForCausalLM):
|
||||
)
|
||||
|
||||
def pad_input_ids(self, input_ids: array, mm_inputs: MultimodalInputs) -> array:
|
||||
return MultiModalityDataPaddingPatternMultimodalTokens().pad_input_tokens(
|
||||
input_ids, mm_inputs
|
||||
)
|
||||
return pad_mm_tokens(input_ids, mm_inputs)
|
||||
|
||||
def get_image_feature(self, items: List[MultimodalDataItem]) -> torch.Tensor:
|
||||
pixel_values = torch.cat([item.feature for item in items], dim=0)
|
||||
|
||||
@@ -28,8 +28,8 @@ from sglang.srt.layers.activation import ReLU2
|
||||
from sglang.srt.layers.layernorm import RMSNorm
|
||||
from sglang.srt.layers.quantization.base_config import QuantizationConfig
|
||||
from sglang.srt.managers.mm_utils import (
|
||||
MultiModalityDataPaddingPatternTokenPairs,
|
||||
general_mm_embed_routine,
|
||||
pad_mm_token_pairs,
|
||||
)
|
||||
from sglang.srt.managers.schedule_batch import (
|
||||
Modality,
|
||||
@@ -149,20 +149,18 @@ class NemotronH_Nano_VL_V2(EVS):
|
||||
|
||||
if visual_items:
|
||||
mm_inputs.mm_items = visual_items
|
||||
helper = MultiModalityDataPaddingPatternTokenPairs(
|
||||
[(im_start_id, im_end_id)]
|
||||
input_ids = pad_mm_token_pairs(
|
||||
input_ids, mm_inputs, data_token_pairs=[(im_start_id, im_end_id)]
|
||||
)
|
||||
input_ids = helper.pad_input_tokens(input_ids, mm_inputs)
|
||||
all_data_offsets.extend(mm_inputs.data_offsets)
|
||||
|
||||
audio_start_id = getattr(mm_inputs, "audio_start_id", None)
|
||||
audio_end_id = getattr(mm_inputs, "audio_end_id", None)
|
||||
if audio_items and audio_start_id is not None and audio_end_id is not None:
|
||||
mm_inputs.mm_items = audio_items
|
||||
helper = MultiModalityDataPaddingPatternTokenPairs(
|
||||
[(audio_start_id, audio_end_id)]
|
||||
input_ids = pad_mm_token_pairs(
|
||||
input_ids, mm_inputs, data_token_pairs=[(audio_start_id, audio_end_id)]
|
||||
)
|
||||
input_ids = helper.pad_input_tokens(input_ids, mm_inputs)
|
||||
all_data_offsets.extend(mm_inputs.data_offsets)
|
||||
|
||||
mm_inputs.mm_items = visual_items + audio_items
|
||||
|
||||
@@ -19,7 +19,7 @@ import sglang.srt.model_loader.weight_utils as weight_utils
|
||||
import sglang.srt.utils as utils
|
||||
from sglang.srt.layers.logits_processor import LogitsProcessorOutput
|
||||
from sglang.srt.layers.quantization.base_config import QuantizationConfig
|
||||
from sglang.srt.managers.mm_utils import MultiModalityDataPaddingPatternMultimodalTokens
|
||||
from sglang.srt.managers.mm_utils import pad_mm_tokens
|
||||
from sglang.srt.managers.schedule_batch import (
|
||||
Modality,
|
||||
MultimodalDataItem,
|
||||
@@ -220,8 +220,7 @@ class NVILAForConditionalGeneration(nn.Module):
|
||||
weight_loader(param, loaded_weight)
|
||||
|
||||
def pad_input_ids(self, input_ids: array, mm_inputs: MultimodalInputs) -> array:
|
||||
pattern = MultiModalityDataPaddingPatternMultimodalTokens()
|
||||
return pattern.pad_input_tokens(input_ids, mm_inputs)
|
||||
return pad_mm_tokens(input_ids, mm_inputs)
|
||||
|
||||
|
||||
def merge_chessboard(x, num_split_h, num_split_w):
|
||||
|
||||
@@ -18,7 +18,7 @@ import sglang.srt.model_loader.weight_utils as weight_utils
|
||||
import sglang.srt.utils as utils
|
||||
from sglang.srt.layers.logits_processor import LogitsProcessorOutput
|
||||
from sglang.srt.layers.quantization.base_config import QuantizationConfig
|
||||
from sglang.srt.managers.mm_utils import MultiModalityDataPaddingPatternMultimodalTokens
|
||||
from sglang.srt.managers.mm_utils import pad_mm_tokens
|
||||
from sglang.srt.managers.schedule_batch import (
|
||||
Modality,
|
||||
MultimodalDataItem,
|
||||
@@ -180,8 +180,7 @@ class NVILALiteForConditionalGeneration(nn.Module):
|
||||
weight_loader(param, loaded_weight)
|
||||
|
||||
def pad_input_ids(self, input_ids: array, mm_inputs: MultimodalInputs) -> array:
|
||||
pattern = MultiModalityDataPaddingPatternMultimodalTokens()
|
||||
return pattern.pad_input_tokens(input_ids, mm_inputs)
|
||||
return pad_mm_tokens(input_ids, mm_inputs)
|
||||
|
||||
|
||||
EntryClass = [NVILALiteForConditionalGeneration]
|
||||
|
||||
@@ -43,8 +43,8 @@ from sglang.srt.layers.conv import Conv2dLayer
|
||||
from sglang.srt.layers.linear import ColumnParallelLinear, RowParallelLinear
|
||||
from sglang.srt.layers.quantization.base_config import QuantizationConfig
|
||||
from sglang.srt.managers.mm_utils import (
|
||||
MultiModalityDataPaddingPatternMultimodalTokens,
|
||||
general_mm_embed_routine,
|
||||
pad_mm_tokens,
|
||||
)
|
||||
from sglang.srt.managers.schedule_batch import MultimodalDataItem, MultimodalInputs
|
||||
from sglang.srt.model_executor.forward_batch_info import ForwardBatch
|
||||
@@ -586,8 +586,7 @@ class PaddleOCRVLForConditionalGeneration(Ernie4_5_ForCausalLM):
|
||||
self.is_mrope_enabled = "mrope_section" in (self.config.rope_scaling or {})
|
||||
|
||||
def pad_input_ids(self, input_ids: array, mm_inputs: MultimodalInputs) -> array:
|
||||
pattern = MultiModalityDataPaddingPatternMultimodalTokens()
|
||||
return pattern.pad_input_tokens(input_ids, mm_inputs)
|
||||
return pad_mm_tokens(input_ids, mm_inputs)
|
||||
|
||||
def get_input_embeddings(self):
|
||||
return self.model.embed_tokens
|
||||
|
||||
@@ -31,8 +31,8 @@ from transformers import PretrainedConfig
|
||||
|
||||
from sglang.srt.layers.quantization import QuantizationConfig
|
||||
from sglang.srt.managers.mm_utils import (
|
||||
MultiModalityDataPaddingPatternMultimodalTokens,
|
||||
general_mm_embed_routine,
|
||||
pad_mm_tokens,
|
||||
)
|
||||
from sglang.srt.managers.schedule_batch import (
|
||||
Modality,
|
||||
@@ -477,8 +477,7 @@ class Phi4MMForCausalLM(nn.Module):
|
||||
return hidden_states
|
||||
|
||||
def pad_input_ids(self, input_ids: array, mm_inputs: MultimodalInputs) -> array:
|
||||
pattern = MultiModalityDataPaddingPatternMultimodalTokens()
|
||||
return pattern.pad_input_tokens(input_ids, mm_inputs)
|
||||
return pad_mm_tokens(input_ids, mm_inputs)
|
||||
|
||||
def should_apply_lora(self, module_name: str) -> bool:
|
||||
return bool(self.lora_pattern.match(module_name))
|
||||
|
||||
@@ -38,8 +38,8 @@ from sglang.srt.layers.layernorm import RMSNorm
|
||||
from sglang.srt.layers.linear import MergedColumnParallelLinear, RowParallelLinear
|
||||
from sglang.srt.layers.quantization.base_config import QuantizationConfig
|
||||
from sglang.srt.managers.mm_utils import (
|
||||
MultiModalityDataPaddingPatternMultimodalTokens,
|
||||
general_mm_embed_routine,
|
||||
pad_mm_tokens,
|
||||
)
|
||||
from sglang.srt.managers.schedule_batch import MultimodalDataItem, MultimodalInputs
|
||||
from sglang.srt.model_loader.weight_utils import default_weight_loader
|
||||
@@ -136,8 +136,7 @@ class PixtralForConditionalGeneration(nn.Module):
|
||||
)
|
||||
|
||||
def pad_input_ids(self, input_ids: array, mm_inputs: MultimodalInputs) -> array:
|
||||
pattern = MultiModalityDataPaddingPatternMultimodalTokens()
|
||||
return pattern.pad_input_tokens(input_ids, mm_inputs)
|
||||
return pad_mm_tokens(input_ids, mm_inputs)
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]):
|
||||
def is_vision_encoder_weights(weight: tuple[str, torch.Tensor]):
|
||||
@@ -857,7 +856,7 @@ class PixtralHFVisionModel(nn.Module):
|
||||
DEFAULT_IMAGE_TOKEN_ID = 10
|
||||
|
||||
def pad_input_ids(self, input_ids: array, mm_inputs: MultimodalInputs) -> array:
|
||||
return self.input_padder.pad_input_tokens(input_ids, mm_inputs)
|
||||
return pad_mm_tokens(input_ids, mm_inputs)
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
@@ -902,7 +901,6 @@ class PixtralHFVisionModel(nn.Module):
|
||||
|
||||
# Initialize patch position embedding
|
||||
self.patch_positional_embedding = PixtralVisionRotaryEmbedding(config)
|
||||
self.input_padder = MultiModalityDataPaddingPatternMultimodalTokens()
|
||||
|
||||
@property
|
||||
def dtype(self):
|
||||
|
||||
@@ -9,8 +9,8 @@ from torch import nn
|
||||
from sglang.srt.configs.points_v15_chat import POINTSV15ChatConfig
|
||||
from sglang.srt.layers.quantization.base_config import QuantizationConfig
|
||||
from sglang.srt.managers.mm_utils import (
|
||||
MultiModalityDataPaddingPatternMultimodalTokens,
|
||||
general_mm_embed_routine,
|
||||
pad_mm_tokens,
|
||||
)
|
||||
from sglang.srt.managers.schedule_batch import (
|
||||
Modality,
|
||||
@@ -103,8 +103,7 @@ class POINTSV15ChatModel(nn.Module):
|
||||
)
|
||||
|
||||
def pad_input_ids(self, input_ids: array, mm_inputs: MultimodalInputs) -> array:
|
||||
pattern = MultiModalityDataPaddingPatternMultimodalTokens()
|
||||
return pattern.pad_input_tokens(input_ids, mm_inputs)
|
||||
return pad_mm_tokens(input_ids, mm_inputs)
|
||||
|
||||
def get_image_feature(self, items: List[MultimodalDataItem]) -> torch.Tensor:
|
||||
pixel_values = torch.cat([item.feature for item in items], dim=0).type(
|
||||
|
||||
@@ -61,8 +61,8 @@ from sglang.srt.layers.quantization.base_config import QuantizationConfig
|
||||
from sglang.srt.layers.utils import PPMissingLayer, get_layer_id
|
||||
from sglang.srt.layers.vocab_parallel_embedding import ParallelLMHead
|
||||
from sglang.srt.managers.mm_utils import (
|
||||
MultiModalityDataPaddingPatternMultimodalTokens,
|
||||
general_mm_embed_routine,
|
||||
pad_mm_tokens,
|
||||
)
|
||||
from sglang.srt.managers.schedule_batch import (
|
||||
Modality,
|
||||
@@ -760,8 +760,7 @@ class Qwen2_5_VLForConditionalGeneration(nn.Module):
|
||||
self.capture_aux_hidden_states = False
|
||||
|
||||
def pad_input_ids(self, input_ids: array, mm_inputs: MultimodalInputs) -> array:
|
||||
pattern = MultiModalityDataPaddingPatternMultimodalTokens()
|
||||
return pattern.pad_input_tokens(input_ids, mm_inputs)
|
||||
return pad_mm_tokens(input_ids, mm_inputs)
|
||||
|
||||
def get_image_feature(self, items: List[MultimodalDataItem]) -> torch.Tensor:
|
||||
# in qwen-vl, last dim is the same
|
||||
|
||||
@@ -38,8 +38,8 @@ from transformers.models.qwen2_audio.modeling_qwen2_audio import (
|
||||
|
||||
from sglang.srt.layers.quantization.base_config import QuantizationConfig
|
||||
from sglang.srt.managers.mm_utils import (
|
||||
MultiModalityDataPaddingPatternMultimodalTokens,
|
||||
general_mm_embed_routine,
|
||||
pad_mm_tokens,
|
||||
)
|
||||
from sglang.srt.managers.schedule_batch import (
|
||||
Modality,
|
||||
@@ -99,10 +99,9 @@ class Qwen2AudioForConditionalGeneration(nn.Module):
|
||||
self.language_model = Qwen2ForCausalLM(
|
||||
config.text_config, quant_config, prefix=add_prefix("model", prefix)
|
||||
)
|
||||
self.pattern = MultiModalityDataPaddingPatternMultimodalTokens()
|
||||
|
||||
def pad_input_ids(self, input_ids: array, mm_inputs: MultimodalInputs) -> array:
|
||||
return self.pattern.pad_input_tokens(input_ids, mm_inputs)
|
||||
return pad_mm_tokens(input_ids, mm_inputs)
|
||||
|
||||
def get_audio_feature(self, items: List[MultimodalDataItem]) -> torch.Tensor:
|
||||
# Extract audio features from input items
|
||||
|
||||
@@ -47,8 +47,8 @@ from sglang.srt.layers.pooler import Pooler, PoolingType
|
||||
from sglang.srt.layers.quantization.base_config import QuantizationConfig
|
||||
from sglang.srt.layers.vocab_parallel_embedding import ParallelLMHead
|
||||
from sglang.srt.managers.mm_utils import (
|
||||
MultiModalityDataPaddingPatternMultimodalTokens,
|
||||
general_mm_embed_routine,
|
||||
pad_mm_tokens,
|
||||
)
|
||||
from sglang.srt.managers.schedule_batch import MultimodalDataItem, MultimodalInputs
|
||||
from sglang.srt.model_executor.forward_batch_info import ForwardBatch
|
||||
@@ -494,8 +494,7 @@ class Qwen2VLForConditionalGeneration(nn.Module):
|
||||
self.pooler = Pooler(pooling_type=PoolingType.LAST, normalize=True)
|
||||
|
||||
def pad_input_ids(self, input_ids: array, mm_inputs: MultimodalInputs) -> array:
|
||||
pattern = MultiModalityDataPaddingPatternMultimodalTokens()
|
||||
return pattern.pad_input_tokens(input_ids, mm_inputs)
|
||||
return pad_mm_tokens(input_ids, mm_inputs)
|
||||
|
||||
def get_image_feature(self, items: List[MultimodalDataItem]) -> torch.Tensor:
|
||||
# in qwen-vl, last dim is the same
|
||||
|
||||
@@ -11,8 +11,8 @@ from sglang.srt.configs.qwen3_asr import Qwen3ASRConfig
|
||||
from sglang.srt.configs.qwen3_omni import Qwen3OmniMoeAudioEncoderConfig
|
||||
from sglang.srt.layers.quantization.base_config import QuantizationConfig
|
||||
from sglang.srt.managers.mm_utils import (
|
||||
MultiModalityDataPaddingPatternMultimodalTokens,
|
||||
general_mm_embed_routine,
|
||||
pad_mm_tokens,
|
||||
)
|
||||
from sglang.srt.managers.schedule_batch import (
|
||||
Modality,
|
||||
@@ -65,10 +65,9 @@ class Qwen3ASRForConditionalGeneration(nn.Module):
|
||||
quant_config,
|
||||
prefix=add_prefix("language_model", prefix),
|
||||
)
|
||||
self.pattern = MultiModalityDataPaddingPatternMultimodalTokens()
|
||||
|
||||
def pad_input_ids(self, input_ids: array, mm_inputs: MultimodalInputs) -> array:
|
||||
return self.pattern.pad_input_tokens(input_ids, mm_inputs)
|
||||
return pad_mm_tokens(input_ids, mm_inputs)
|
||||
|
||||
def get_audio_feature(self, items: List[MultimodalDataItem]) -> torch.Tensor:
|
||||
device = next(self.audio_tower.parameters()).device
|
||||
|
||||
@@ -60,8 +60,8 @@ from sglang.srt.layers.vocab_parallel_embedding import (
|
||||
VocabParallelEmbedding,
|
||||
)
|
||||
from sglang.srt.managers.mm_utils import (
|
||||
MultiModalityDataPaddingPatternMultimodalTokens,
|
||||
general_mm_embed_routine,
|
||||
pad_mm_tokens,
|
||||
)
|
||||
from sglang.srt.managers.schedule_batch import (
|
||||
Modality,
|
||||
@@ -1437,8 +1437,7 @@ class Qwen3VLForConditionalGeneration(nn.Module):
|
||||
def pad_input_ids(self, input_ids: array, mm_inputs: MultimodalInputs) -> array:
|
||||
if mm_inputs and mm_inputs.mm_items:
|
||||
_require_vision(self)
|
||||
pattern = MultiModalityDataPaddingPatternMultimodalTokens()
|
||||
return pattern.pad_input_tokens(input_ids, mm_inputs)
|
||||
return pad_mm_tokens(input_ids, mm_inputs)
|
||||
|
||||
def get_image_feature(self, items: List[MultimodalDataItem]) -> torch.Tensor:
|
||||
_require_vision(self)
|
||||
|
||||
@@ -25,8 +25,8 @@ from sglang.srt.layers.logits_processor import LogitsProcessor
|
||||
from sglang.srt.layers.pooler import Pooler, PoolingType
|
||||
from sglang.srt.layers.quantization.base_config import QuantizationConfig
|
||||
from sglang.srt.managers.mm_utils import (
|
||||
MultiModalityDataPaddingPatternMultimodalTokens,
|
||||
general_mm_embed_routine,
|
||||
pad_mm_tokens,
|
||||
)
|
||||
from sglang.srt.managers.schedule_batch import MultimodalDataItem, MultimodalInputs
|
||||
from sglang.srt.model_executor.forward_batch_info import ForwardBatch
|
||||
@@ -110,8 +110,7 @@ class Sarashina2VisionForCausalLM(nn.Module):
|
||||
|
||||
def pad_input_ids(self, input_ids: array, mm_inputs: MultimodalInputs) -> array:
|
||||
"""Pad input tokens with multimodal data hashes for RadixAttention."""
|
||||
pattern = MultiModalityDataPaddingPatternMultimodalTokens()
|
||||
return pattern.pad_input_tokens(input_ids, mm_inputs)
|
||||
return pad_mm_tokens(input_ids, mm_inputs)
|
||||
|
||||
def get_input_embeddings(self):
|
||||
"""Get input embeddings from the language model."""
|
||||
|
||||
@@ -50,8 +50,8 @@ from sglang.srt.layers.vocab_parallel_embedding import (
|
||||
VocabParallelEmbedding,
|
||||
)
|
||||
from sglang.srt.managers.mm_utils import (
|
||||
MultiModalityDataPaddingPatternMultimodalTokens,
|
||||
general_mm_embed_routine,
|
||||
pad_mm_tokens,
|
||||
)
|
||||
from sglang.srt.managers.schedule_batch import (
|
||||
Modality,
|
||||
@@ -885,8 +885,7 @@ class Step3VLForConditionalGeneration(nn.Module):
|
||||
return self._flatten_embeddings(merged_image_features)
|
||||
|
||||
def pad_input_ids(self, input_ids: array, mm_inputs: MultimodalInputs) -> array:
|
||||
pattern = MultiModalityDataPaddingPatternMultimodalTokens()
|
||||
return pattern.pad_input_tokens(input_ids, mm_inputs)
|
||||
return pad_mm_tokens(input_ids, mm_inputs)
|
||||
|
||||
@torch.no_grad()
|
||||
def forward(
|
||||
|
||||
@@ -18,8 +18,8 @@ from sglang.srt.layers.conv import Conv2dLayer
|
||||
from sglang.srt.layers.linear import ColumnParallelLinear, RowParallelLinear
|
||||
from sglang.srt.layers.quantization.base_config import QuantizationConfig
|
||||
from sglang.srt.managers.mm_utils import (
|
||||
MultiModalityDataPaddingPatternMultimodalTokens,
|
||||
general_mm_embed_routine,
|
||||
pad_mm_tokens,
|
||||
)
|
||||
from sglang.srt.managers.schedule_batch import (
|
||||
Modality,
|
||||
@@ -572,8 +572,7 @@ class StepVLForConditionalGeneration(nn.Module):
|
||||
return self._flatten_embeddings(merged_image_features)
|
||||
|
||||
def pad_input_ids(self, input_ids: array, mm_inputs: MultimodalInputs) -> array:
|
||||
pattern = MultiModalityDataPaddingPatternMultimodalTokens()
|
||||
return pattern.pad_input_tokens(input_ids, mm_inputs)
|
||||
return pad_mm_tokens(input_ids, mm_inputs)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
|
||||
@@ -9,8 +9,8 @@ from sglang.srt.configs.step3p7 import Step3p7Config
|
||||
from sglang.srt.layers.linear import ColumnParallelLinear
|
||||
from sglang.srt.layers.quantization.base_config import QuantizationConfig
|
||||
from sglang.srt.managers.mm_utils import (
|
||||
MultiModalityDataPaddingPatternMultimodalTokens,
|
||||
general_mm_embed_routine,
|
||||
pad_mm_tokens,
|
||||
)
|
||||
from sglang.srt.managers.schedule_batch import (
|
||||
Modality,
|
||||
@@ -130,8 +130,7 @@ class Step3p7ForConditionalGeneration(nn.Module):
|
||||
return self._flatten_embeddings(merged_image_features)
|
||||
|
||||
def pad_input_ids(self, input_ids: array, mm_inputs: MultimodalInputs) -> array:
|
||||
pattern = MultiModalityDataPaddingPatternMultimodalTokens()
|
||||
return pattern.pad_input_tokens(input_ids, mm_inputs)
|
||||
return pad_mm_tokens(input_ids, mm_inputs)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
|
||||
@@ -58,9 +58,6 @@ from sglang.srt.layers.vocab_parallel_embedding import (
|
||||
ParallelLMHead,
|
||||
VocabParallelEmbedding,
|
||||
)
|
||||
from sglang.srt.managers.mm_utils import (
|
||||
MultiModalityDataPaddingPatternMultimodalTokens,
|
||||
)
|
||||
from sglang.srt.managers.schedule_batch import MultimodalDataItem, MultimodalInputs
|
||||
from sglang.srt.model_executor.forward_batch_info import ForwardBatch, PPProxyTensors
|
||||
from sglang.srt.model_loader.weight_utils import default_weight_loader
|
||||
@@ -1347,7 +1344,6 @@ class MultiModalMixin:
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
super().__init__(*args, **kwargs)
|
||||
self._mm_padding_pattern = MultiModalityDataPaddingPatternMultimodalTokens()
|
||||
|
||||
# transformers v5 flattened SigLIP/CLIP (dropped the "vision_model"
|
||||
# wrapper); older checkpoints still ship "vision_tower.vision_model.*"
|
||||
|
||||
@@ -10,8 +10,8 @@ from torch import Tensor, nn
|
||||
from sglang.srt.configs.unlimited_ocr import UnlimitedVLConfig
|
||||
from sglang.srt.layers.quantization import QuantizationConfig
|
||||
from sglang.srt.managers.mm_utils import (
|
||||
MultiModalityDataPaddingPatternMultimodalTokens,
|
||||
general_mm_embed_routine,
|
||||
pad_mm_tokens,
|
||||
)
|
||||
from sglang.srt.managers.schedule_batch import MultimodalDataItem, MultimodalInputs
|
||||
from sglang.srt.model_executor.forward_batch_info import ForwardBatch
|
||||
@@ -314,8 +314,7 @@ class UnlimitedOCRForCausalLM(nn.Module):
|
||||
|
||||
def pad_input_ids(self, input_ids: array, mm_inputs: MultimodalInputs) -> array:
|
||||
"""Pad input token IDs with multimodal placeholder tokens."""
|
||||
pattern = MultiModalityDataPaddingPatternMultimodalTokens()
|
||||
return pattern.pad_input_tokens(input_ids, mm_inputs)
|
||||
return pad_mm_tokens(input_ids, mm_inputs)
|
||||
|
||||
def get_image_feature(self, items: List[MultimodalDataItem]) -> torch.Tensor:
|
||||
"""Extract vision features from multimodal data items."""
|
||||
|
||||
@@ -24,8 +24,8 @@ from sglang.srt.layers.linear import (
|
||||
)
|
||||
from sglang.srt.layers.quantization.base_config import QuantizationConfig
|
||||
from sglang.srt.managers.mm_utils import (
|
||||
MultiModalityDataPaddingPatternMultimodalTokens,
|
||||
general_mm_embed_routine,
|
||||
pad_mm_tokens,
|
||||
)
|
||||
from sglang.srt.managers.schedule_batch import (
|
||||
Modality,
|
||||
@@ -243,8 +243,6 @@ class VoxtralForConditionalGeneration(nn.Module):
|
||||
# Mel filter bank for raw waveform -> mel spectrogram
|
||||
self._init_mel_filters(audio_config)
|
||||
|
||||
self.pattern = MultiModalityDataPaddingPatternMultimodalTokens()
|
||||
|
||||
def _init_mel_filters(self, audio_config: PretrainedConfig):
|
||||
"""Initialize mel filter bank for mel spectrogram computation."""
|
||||
self._window_size = getattr(audio_config, "window_size", 400)
|
||||
@@ -339,7 +337,7 @@ class VoxtralForConditionalGeneration(nn.Module):
|
||||
return results
|
||||
|
||||
def pad_input_ids(self, input_ids: array, mm_inputs: MultimodalInputs) -> array:
|
||||
return self.pattern.pad_input_tokens(input_ids, mm_inputs)
|
||||
return pad_mm_tokens(input_ids, mm_inputs)
|
||||
|
||||
def get_audio_feature(self, items: List[MultimodalDataItem]) -> torch.Tensor:
|
||||
"""Encode audio waveforms -> downsample -> project."""
|
||||
|
||||
Reference in New Issue
Block a user