[MM] Keep scheduler padding in packed token arrays (#40357)

This commit is contained in:
Jialin Ouyang
2026-09-22 15:12:07 -07:00
committed by GitHub
parent c19dc43cc7
commit b77833c504
64 changed files with 376 additions and 110 deletions
+35 -17
View File
@@ -9,9 +9,10 @@ import os
import pickle
import sys
from abc import abstractmethod
from array import array
from collections import defaultdict
from multiprocessing import shared_memory
from typing import Any, Dict, List, Optional, Tuple
from typing import Any, Callable, Dict, List, Optional, Tuple
import numpy as np
import torch
@@ -49,7 +50,11 @@ from sglang.srt.runtime_context import (
get_server_args,
get_serving,
)
from sglang.srt.utils import flatten_nested_list, print_warning_once
from sglang.srt.utils import (
assert_int64_array,
flatten_nested_list,
print_warning_once,
)
from sglang.srt.utils.stale_shm_cleanup import make_shm_name
from sglang.utils import logger
@@ -232,6 +237,19 @@ class TransportProxyTensor(torch.Tensor):
return self._metadata.get("transport_mode", "default")
def duplicate_pad_mm_input_ids(
input_ids: array,
mm_inputs: MultimodalInputs,
pad_input_ids_func: Callable[[array, MultimodalInputs], array],
) -> array:
"""Validate and duplicate model-padded tokens for scheduler ownership."""
padded_ids = pad_input_ids_func(input_ids, mm_inputs)
assert_int64_array(padded_ids, "pad_input_ids result")
# Preserve scheduler ownership: no-op models return the unpadded input,
# and other models can retain the result in multimodal metadata.
return padded_ids[:]
class MultiModalityDataPaddingPattern:
"""
Data tokens (like image tokens) often need special handling during padding
@@ -240,11 +258,10 @@ class MultiModalityDataPaddingPattern:
"""
@abstractmethod
def pad_input_tokens(
self, input_ids: List[int], mm_inputs: MultimodalInputs
) -> List[int]:
def pad_input_tokens(self, input_ids: array, mm_inputs: MultimodalInputs) -> array:
"""
Pad the input ids sequence containing data tokens, and replace them with pad_values
Replace data tokens with pad_values using native int64 arrays.
Implementations must not mutate the original input.
"""
pass
@@ -273,12 +290,11 @@ class MultiModalityDataPaddingPatternTokenPairs(MultiModalityDataPaddingPattern)
s for s, _e in data_token_pairs
]
def pad_input_tokens(
self, input_ids: List[int], mm_inputs: MultimodalInputs
) -> List[int]:
def pad_input_tokens(self, input_ids: array, mm_inputs: MultimodalInputs) -> array:
"""
This function will replace the data-tokens in between with pad_values accordingly
"""
assert_int64_array(input_ids, "input_ids")
pad_values = [item.pad_value for item in mm_inputs.mm_items]
data_token_pairs = self.data_token_id_pairs
mm_inputs.data_offsets = []
@@ -292,7 +308,7 @@ class MultiModalityDataPaddingPatternTokenPairs(MultiModalityDataPaddingPattern)
start_token_ids = {s for s, _e in data_token_pairs}
end_tokens_ids = {e for _s, e in data_token_pairs}
padded_ids = []
padded_ids = array("q")
last_idx = 0
data_idx = -1
@@ -314,7 +330,8 @@ class MultiModalityDataPaddingPatternTokenPairs(MultiModalityDataPaddingPattern)
num_tokens = end_idx - start_idx - 1
pad_value = pad_values[data_idx]
padded_ids.extend([pad_value] * num_tokens)
if num_tokens > 0:
padded_ids.extend(array("q", [pad_value]) * num_tokens)
last_idx = end_idx
@@ -329,17 +346,19 @@ class MultiModalityDataPaddingPatternMultimodalTokens(MultiModalityDataPaddingPa
e.g. <image><image>....<image>, or <audio><audio>...<audio>
"""
def pad_input_tokens(
self, input_ids: List[int], mm_inputs: MultimodalInputs
) -> List[int]:
def pad_input_tokens(self, input_ids: array, mm_inputs: MultimodalInputs) -> array:
"""
Replaces multimodal tokens in input_ids with corresponding pad_values from mm_items.
Each modality (image, audio, video) is handled separately based on its token_id.
"""
assert_int64_array(input_ids, "input_ids")
if not input_ids or not mm_inputs.mm_items:
return input_ids
input_ids_tensor = torch.as_tensor(input_ids)
# as_tensor(array) walks and boxes every token. View a bulk copy instead,
# keeping the original request's unpadded token buffer unchanged.
padded_input_ids = input_ids[:]
input_ids_tensor = torch.frombuffer(padded_input_ids, dtype=torch.int64)
# Replace multimodal tokens using per-item offsets
items_by_modality = defaultdict(list)
@@ -362,8 +381,7 @@ class MultiModalityDataPaddingPatternMultimodalTokens(MultiModalityDataPaddingPa
for offset in items[i].offsets:
input_ids_tensor[offset[0] : offset[1] + 1] = item.pad_value
ret_input_ids = input_ids_tensor.tolist()
return ret_input_ids
return padded_input_ids
# masked_scatter_ materializes the expanded [num_tokens, hidden] bool mask plus
+5 -6
View File
@@ -193,6 +193,7 @@ from sglang.srt.managers.min_free_slots_delayer import (
MinFreeSlotsDelayer,
resolve_min_free_slots,
)
from sglang.srt.managers.mm_utils import duplicate_pad_mm_input_ids
from sglang.srt.managers.multimodal_processor import get_mm_processor, import_processors
from sglang.srt.managers.overlap_utils import (
RelayPayload,
@@ -1214,7 +1215,6 @@ class Scheduler(
)
self.dp_tp_cpu_group = self.dp_tp_group.cpu_group
# TODO(Jialin): Migrate pad_input_ids implementations to return array.
self.pad_input_ids_func = self.tp_worker.get_pad_input_ids_func()
set_random_seed(self.random_seed)
@@ -2994,8 +2994,8 @@ class Scheduler(
not self._try_apply_padded_mm_input_ids(recv_req, req, image_inputs)
and self.pad_input_ids_func
):
req.origin_input_ids = array(
"q", self.pad_input_ids_func(req.origin_input_ids, image_inputs)
req.origin_input_ids = duplicate_pad_mm_input_ids(
req.origin_input_ids, image_inputs, self.pad_input_ids_func
)
req.extend_image_inputs(image_inputs)
self._maybe_compute_mrope_positions(req)
@@ -3468,9 +3468,8 @@ class Scheduler(
not self._try_apply_padded_mm_input_ids(recv_req, req, image_inputs)
and self.pad_input_ids_func
):
# See companion call site above for the array.array wrap rationale.
req.origin_input_ids = array(
"q", self.pad_input_ids_func(req.origin_input_ids, image_inputs)
req.origin_input_ids = duplicate_pad_mm_input_ids(
req.origin_input_ids, image_inputs, self.pad_input_ids_func
)
req.extend_image_inputs(image_inputs)
@@ -55,6 +55,7 @@ from sglang.srt.mem_cache.unified_cache.unified_tree_core_interface import (
UnifiedTreeCoreInterface,
)
from sglang.srt.runtime_context import get_exec, mamba_cache_chunk_size
from sglang.srt.utils import assert_int64_array
if TYPE_CHECKING:
from sglang.srt.managers.schedule_batch import Req
@@ -72,9 +73,7 @@ def _radix_key_buffer(key: RadixKey) -> array:
"""The key's token ids honoring `limit`; view-independent since the
binding derives its own atoms."""
token_ids = key.raw_token_ids()
assert isinstance(token_ids, array) and token_ids.typecode == "q", (
f"tree keys must carry array('q') token ids, got {type(token_ids).__name__}"
)
assert_int64_array(token_ids, "tree key token ids")
return token_ids
+2 -1
View File
@@ -14,6 +14,7 @@
"""Legacy Bailing multimodal wrappers for image and video inference."""
import logging
from array import array
from typing import Iterable, List, Optional, Set, Tuple
import torch
@@ -122,7 +123,7 @@ class BailingMMNativeForConditionalGeneration(nn.Module):
def get_input_embeddings(self):
return self.model.get_input_embeddings()
def pad_input_ids(self, input_ids: List[int], mm_inputs: MultimodalInputs):
def pad_input_ids(self, input_ids: array, mm_inputs: MultimodalInputs) -> array:
return self.pattern.pad_input_tokens(input_ids, mm_inputs)
def _get_vision_feature(
+2 -1
View File
@@ -14,6 +14,7 @@
"""SGLang implementation of Bailing/Ling 3 VL image and video inference."""
import logging
from array import array
from typing import Iterable, List, Optional, Set, Tuple
import torch
@@ -128,7 +129,7 @@ class BailingMoeV3VLForConditionalGeneration(nn.Module):
def get_input_embeddings(self):
return self.model.get_input_embeddings()
def pad_input_ids(self, input_ids: List[int], mm_inputs: MultimodalInputs):
def pad_input_ids(self, input_ids: array, mm_inputs: MultimodalInputs) -> array:
return self.pattern.pad_input_tokens(input_ids, mm_inputs)
def _materialize_items(self, items: List[MultimodalDataItem]) -> torch.Tensor:
+2 -1
View File
@@ -1,6 +1,7 @@
# Adapted from
# https://github.com/huggingface/transformers/blob/af9b2eaa54c150741f298d6db939af6328e1dc38/src/transformers/models/clip/modeling_clip.py
from array import array
from functools import partial
from typing import Iterable, List, Optional, Tuple, Type, Union
@@ -580,7 +581,7 @@ class CLIPModel(nn.Module):
embeddings=self.text_projection(pooled_output.embeddings)
)
def pad_input_ids(self, input_ids: List[int], image_inputs: MultimodalInputs):
def pad_input_ids(self, input_ids: array, image_inputs: MultimodalInputs) -> array:
# Clip embeddings models handle text/image separately, so we don't need to pad input ids
return input_ids
+2 -3
View File
@@ -5,6 +5,7 @@
"""Inference-only Cohere2Vision (Command-A-Vision) multimodal model."""
import math
from array import array
from typing import Iterable, List, Optional, Tuple
import torch
@@ -151,9 +152,7 @@ class Cohere2VisionForConditionalGeneration(nn.Module):
# walks ``model.model.layers``) can locate the transformer layers.
self.model = self.language_model.model
def pad_input_ids(
self, input_ids: List[int], mm_inputs: MultimodalInputs
) -> List[int]:
def pad_input_ids(self, input_ids: array, mm_inputs: MultimodalInputs) -> array:
pattern = MultiModalityDataPaddingPatternMultimodalTokens()
return pattern.pad_input_tokens(input_ids, mm_inputs)
+2 -3
View File
@@ -19,6 +19,7 @@ Arcee causal-LM structure used by SGLang, while the vision path uses the native
SigLIP2 implementation and an Edge-specific spatial-merge projector.
"""
from array import array
from typing import Iterable, List, Optional, Tuple
import numpy as np
@@ -212,9 +213,7 @@ class Cosmos3EdgeForConditionalGeneration(ArceeForCausalLM):
prefix=add_prefix("projector", prefix),
)
def pad_input_ids(
self, input_ids: List[int], mm_inputs: MultimodalInputs
) -> List[int]:
def pad_input_ids(self, input_ids: array, mm_inputs: MultimodalInputs) -> array:
pattern = MultiModalityDataPaddingPatternMultimodalTokens()
return pattern.pad_input_tokens(input_ids, mm_inputs)
@@ -19,6 +19,7 @@
import collections
import math
import os
from array import array
from dataclasses import field
from enum import Enum
from functools import partial
@@ -1995,7 +1996,7 @@ class MultiModalityCausalLM(MultiModalityPreTrainedModel):
def prepare_gen_img_embeds(self, image_ids: torch.LongTensor):
return self.gen_aligner(self.gen_embed(image_ids))
def pad_input_ids(self, input_ids: List[int], image_inputs: MultimodalInputs):
def pad_input_ids(self, input_ids: array, image_inputs: MultimodalInputs) -> array:
im_start_id = image_inputs.im_start_id
im_end_id = image_inputs.im_end_id
media_token_pairs = [(im_start_id, im_end_id)]
+2 -1
View File
@@ -22,6 +22,7 @@
import copy
import logging
import math
from array import array
from functools import partial
from typing import Iterable, List, Optional, Set, Tuple, Type, TypeAlias, Union
@@ -1763,7 +1764,7 @@ class DeepseekOCRForCausalLM(nn.Module):
return inputs_embeds
def pad_input_ids(self, input_ids: List[int], mm_inputs: MultimodalInputs):
def pad_input_ids(self, input_ids: array, mm_inputs: MultimodalInputs) -> array:
pattern = MultiModalityDataPaddingPatternMultimodalTokens()
return pattern.pad_input_tokens(input_ids, mm_inputs)
+2 -1
View File
@@ -4,6 +4,7 @@ import concurrent.futures
import functools
import logging
import time
from array import array
from contextlib import contextmanager, nullcontext
from types import SimpleNamespace
from typing import (
@@ -4981,7 +4982,7 @@ class DeepseekV4ForCausalLM(nn.Module):
def routed_experts_weights_of_layer(self):
return self._routed_experts_weights_of_layer.value
def pad_input_ids(self, input_ids, mm_inputs):
def pad_input_ids(self, input_ids: array, mm_inputs) -> array:
return MultiModalityDataPaddingPatternMultimodalTokens().pad_input_tokens(
input_ids, mm_inputs
)
+2 -1
View File
@@ -1,3 +1,4 @@
from array import array
from typing import Iterable, List, Optional, Tuple
import torch
@@ -262,7 +263,7 @@ class DeepseekVL2ForCausalLM(nn.Module):
weights_loader = getattr(param, "weight_loader", default_weight_loader)
weights_loader(param, loaded_weight)
def pad_input_ids(self, input_ids: List[int], mm_inputs: MultimodalInputs):
def pad_input_ids(self, input_ids: array, mm_inputs: MultimodalInputs) -> array:
pattern = MultiModalityDataPaddingPatternMultimodalTokens()
return pattern.pad_input_tokens(input_ids, mm_inputs)
@@ -19,6 +19,7 @@
import concurrent.futures
import logging
import math
from array import array
from dataclasses import dataclass
from enum import IntEnum, auto
from typing import (
@@ -1921,10 +1922,10 @@ class Dots3LanguageModelForCausalLM(nn.Module):
def pad_input_ids(
self,
input_ids: List[int],
input_ids: array,
mm_inputs: MultimodalInputs,
**kwargs,
) -> List[int]:
) -> array:
token_pairs = []
if mm_inputs.im_start_id is not None and mm_inputs.im_end_id is not None:
token_pairs.append((mm_inputs.im_start_id, mm_inputs.im_end_id))
@@ -2637,7 +2638,7 @@ class DotsNoteOmniThinkerForConditionalGeneration(nn.Module):
def get_input_embeddings(self):
return self.language_model.get_input_embeddings()
def pad_input_ids(self, input_ids, mm_inputs, **kwargs):
def pad_input_ids(self, input_ids: array, mm_inputs, **kwargs) -> array:
return self.language_model.pad_input_ids(input_ids, mm_inputs, **kwargs)
def get_image_feature(self, items: List[MultimodalDataItem]) -> torch.Tensor:
+2 -1
View File
@@ -2,6 +2,7 @@
# Adapted from Qwen2.5-VL SGLang implementation
import logging
from array import array
from typing import Iterable, List, Optional, Tuple
import torch
@@ -56,7 +57,7 @@ class DotsOCRForCausalLM(nn.Module):
self.logits_processor = LogitsProcessor(config)
def pad_input_ids(self, input_ids: List[int], mm_inputs: MultimodalInputs):
def pad_input_ids(self, input_ids: array, mm_inputs: MultimodalInputs) -> array:
pattern = MultiModalityDataPaddingPatternMultimodalTokens()
return pattern.pad_input_tokens(input_ids, mm_inputs)
+2 -1
View File
@@ -17,6 +17,7 @@
# limitations under the License.
"""Inference-only Dots-VL model compatible with HuggingFace weights."""
from array import array
from typing import Iterable, List, Optional, Tuple
import torch
@@ -132,7 +133,7 @@ class DotsVLMForCausalLM(nn.Module):
def get_model_config_for_expert_location(cls, config):
return DeepseekV2ForCausalLM.get_model_config_for_expert_location(config)
def pad_input_ids(self, input_ids: List[int], mm_inputs: MultimodalInputs):
def pad_input_ids(self, input_ids: array, mm_inputs: MultimodalInputs) -> array:
"""Pad input_ids with multimodal tokens"""
# Get image token ID for padding pattern
pattern = MultiModalityDataPaddingPatternMultimodalTokens()
+2 -1
View File
@@ -13,6 +13,7 @@
"""Inference-only Ernie45-VL model compatible with HuggingFace weights."""
import logging
from array import array
from functools import lru_cache, partial
from typing import Iterable, List, Optional, Tuple, Type
@@ -588,7 +589,7 @@ class Ernie4_5_VLMoeForConditionalGeneration(nn.Module):
else:
self._visual_token_ids_tensor_cache = None
def pad_input_ids(self, input_ids: List[int], mm_inputs: MultimodalInputs):
def pad_input_ids(self, input_ids: array, mm_inputs: MultimodalInputs) -> array:
pattern = MultiModalityDataPaddingPatternMultimodalTokens()
return pattern.pad_input_tokens(input_ids, mm_inputs)
+2 -3
View File
@@ -19,6 +19,7 @@
import logging
import re
from array import array
from functools import lru_cache
from typing import Iterable, List, Optional, Set, Tuple, TypedDict
@@ -201,9 +202,7 @@ class Gemma3ForConditionalGeneration(PreTrainedModel):
self.language_model.logits_processor.logit_scale *= logit_scale
self.post_init()
def pad_input_ids(
self, input_ids: List[int], image_inputs: MultimodalInputs
) -> List[int]:
def pad_input_ids(self, input_ids: array, image_inputs: MultimodalInputs) -> array:
"""Pad input IDs with image tokens."""
# Get special token IDs
im_start_id: int = image_inputs.im_start_id
+3 -2
View File
@@ -1,5 +1,6 @@
import logging
import re
from array import array
from functools import lru_cache
from typing import Iterable, List, Optional, Set, Tuple, TypedDict, Union
@@ -243,9 +244,9 @@ class Gemma3nForConditionalGeneration(PreTrainedModel):
def pad_input_ids(
self,
input_ids: List[int],
input_ids: array,
mm_inputs: MultimodalInputs,
) -> List[int]:
) -> array:
"""Pad input IDs with image and audio tokens."""
pattern = MultiModalityDataPaddingPatternMultimodalTokens()
return pattern.pad_input_tokens(input_ids, mm_inputs)
+3 -2
View File
@@ -15,6 +15,7 @@
import logging
import re
from array import array
from functools import lru_cache
from typing import Iterable, List, Optional, Set, Tuple, TypedDict, Union
@@ -296,9 +297,9 @@ class Gemma4ForConditionalGeneration(PreTrainedModel):
def pad_input_ids(
self,
input_ids: List[int],
input_ids: array,
mm_inputs: MultimodalInputs,
) -> List[int]:
) -> array:
"""Pad input IDs with image and audio tokens."""
pattern = MultiModalityDataPaddingPatternMultimodalTokens()
return pattern.pad_input_tokens(input_ids, mm_inputs)
+2 -1
View File
@@ -18,6 +18,7 @@
"""Inference-only GLM-4.1V model compatible with HuggingFace weights."""
import logging
from array import array
from functools import lru_cache
from typing import Iterable, List, Optional, Tuple
@@ -594,7 +595,7 @@ class Glm4vForConditionalGeneration(nn.Module):
# For EAGLE3 support
self.capture_aux_hidden_states = False
def pad_input_ids(self, input_ids: List[int], mm_inputs: MultimodalInputs):
def pad_input_ids(self, input_ids: array, mm_inputs: MultimodalInputs) -> array:
pattern = MultiModalityDataPaddingPatternMultimodalTokens()
return pattern.pad_input_tokens(input_ids, mm_inputs)
+2 -1
View File
@@ -1,4 +1,5 @@
import logging
from array import array
from contextlib import nullcontext
from functools import partial
from typing import Any, Dict, Iterable, List, Optional, Tuple, Union
@@ -1419,7 +1420,7 @@ class Glm5NextForConditionalGeneration(nn.Module):
# Capturing before layer k + 1 gives the completed output of layer k.
self.model.layers_to_capture = [val + 1 for val in layer_ids]
def pad_input_ids(self, input_ids: List[int], mm_inputs: MultimodalInputs):
def pad_input_ids(self, input_ids: array, mm_inputs: MultimodalInputs) -> array:
pattern = MultiModalityDataPaddingPatternMultimodalTokens()
return pattern.pad_input_tokens(input_ids, mm_inputs)
+2 -1
View File
@@ -20,6 +20,7 @@
import copy
import logging
from array import array
from typing import Any, Dict, Iterable, List, Optional, Tuple
import torch
@@ -1068,7 +1069,7 @@ class GlmImageForConditionalGeneration(nn.Module):
self.logits_processor = LogitsProcessor(logits_config)
def pad_input_ids(self, input_ids: List[int], mm_inputs: MultimodalInputs):
def pad_input_ids(self, input_ids: array, mm_inputs: MultimodalInputs) -> array:
pattern = MultiModalityDataPaddingPatternMultimodalTokens()
return pattern.pad_input_tokens(input_ids, mm_inputs)
+2 -1
View File
@@ -18,6 +18,7 @@
"""Inference-only GLM-ASR-HF model compatible with HuggingFace weights."""
import logging
from array import array
from typing import Any, Iterable, List, Optional, Tuple
import torch
@@ -88,7 +89,7 @@ class GlmAsrForConditionalGeneration(nn.Module):
)
self.pattern = MultiModalityDataPaddingPatternMultimodalTokens()
def pad_input_ids(self, input_ids: List[int], mm_inputs: MultimodalInputs):
def pad_input_ids(self, input_ids: array, mm_inputs: MultimodalInputs) -> array:
return self.pattern.pad_input_tokens(input_ids, mm_inputs)
def get_audio_feature(self, items: List[MultimodalDataItem]) -> torch.Tensor:
+2 -1
View File
@@ -3,6 +3,7 @@ from __future__ import annotations
import copy
import logging
import re
from array import array
from typing import Iterable, Optional, Set, Tuple
import torch
@@ -1122,7 +1123,7 @@ class InklingForConditionalGeneration(nn.Module):
return 2
return 1
def pad_input_ids(self, input_ids: list[int], mm_inputs: MultimodalInputs):
def pad_input_ids(self, input_ids: array, mm_inputs: MultimodalInputs) -> array:
# The processor expands one placeholder per media item into a run of the same
# token id; the scheduler calls this to replace each run with the item's
# pad_value (radix hash), which _embed_mm then masks on to scatter the embeds.
+2 -1
View File
@@ -1,3 +1,4 @@
from array import array
from typing import Iterable, List, Optional, Tuple
import torch
@@ -154,7 +155,7 @@ class InternS1ForConditionalGeneration(nn.Module):
return hs
def pad_input_ids(self, input_ids: List[int], mm_inputs: MultimodalInputs):
def pad_input_ids(self, input_ids: array, mm_inputs: MultimodalInputs) -> array:
# Get all special token IDs
im_start_id: int = mm_inputs.im_start_id
im_end_id: int = mm_inputs.im_end_id
+2 -1
View File
@@ -1,5 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from array import array
from typing import Iterable, List, Optional, Tuple, Union
import torch
@@ -674,7 +675,7 @@ class InternVLChatModel(nn.Module):
return hidden_states
def pad_input_ids(self, input_ids: List[int], mm_inputs: MultimodalInputs):
def pad_input_ids(self, input_ids: array, mm_inputs: MultimodalInputs) -> array:
# Get all special token IDs
im_start_id: int = mm_inputs.im_start_id
im_end_id: int = mm_inputs.im_end_id
+2 -3
View File
@@ -1,4 +1,5 @@
import math
from array import array
from collections.abc import Iterable
import einops
@@ -133,9 +134,7 @@ class JetVLMForConditionalGeneration(nn.Module):
)
weight_loader(param, loaded_weight)
def pad_input_ids(
self, input_ids: list[int], mm_inputs: MultimodalInputs
) -> list[int]:
def pad_input_ids(self, input_ids: array, mm_inputs: MultimodalInputs) -> array:
pattern = MultiModalityDataPaddingPatternMultimodalTokens()
return pattern.pad_input_tokens(input_ids, mm_inputs)
+2 -1
View File
@@ -1,4 +1,5 @@
import logging
from array import array
from copy import deepcopy
from typing import Iterable, List, Optional, Sequence, Tuple
@@ -775,7 +776,7 @@ class KimiK25ForConditionalGeneration(nn.Module):
image_embeds = self.vision_tower(pixel_values, grid_thws)
return mm_projection_auto(self.mm_projector, image_embeds)
def pad_input_ids(self, input_ids: List[int], mm_inputs: MultimodalInputs):
def pad_input_ids(self, input_ids: array, mm_inputs: MultimodalInputs) -> array:
pattern = MultiModalityDataPaddingPatternMultimodalTokens()
return pattern.pad_input_tokens(input_ids, mm_inputs)
+2 -1
View File
@@ -9,6 +9,7 @@
import logging
import os
import re
from array import array
from collections.abc import Iterable
from functools import cached_property
from types import SimpleNamespace
@@ -3815,7 +3816,7 @@ class KimiK3ForConditionalGeneration(nn.Module):
image_embeds = self.vision_tower(pixel_values, grid_thws_host.to(device))
return self.mm_projector(image_embeds)
def pad_input_ids(self, input_ids: List[int], mm_inputs: MultimodalInputs):
def pad_input_ids(self, input_ids: array, mm_inputs: MultimodalInputs) -> array:
pattern = MultiModalityDataPaddingPatternMultimodalTokens()
return pattern.pad_input_tokens(input_ids, mm_inputs)
+2 -1
View File
@@ -44,6 +44,7 @@
import copy
import logging
import math
from array import array
from dataclasses import dataclass
from typing import Iterable, List, Optional, Tuple
@@ -192,7 +193,7 @@ class KimiVLForConditionalGeneration(nn.Module):
res = self.multi_modal_projector(image_features)
return res
def pad_input_ids(self, input_ids: List[int], mm_inputs: MultimodalInputs):
def pad_input_ids(self, input_ids: array, mm_inputs: MultimodalInputs) -> array:
pattern = MultiModalityDataPaddingPatternMultimodalTokens()
return pattern.pad_input_tokens(input_ids, mm_inputs)
+2 -3
View File
@@ -20,6 +20,7 @@ LFM2-VL is a vision-language model that combines:
"""
import logging
from array import array
from typing import Iterable, List, Optional, Tuple
import numpy as np
@@ -170,9 +171,7 @@ class Lfm2VlForConditionalGeneration(nn.Module):
self.logits_processor = LogitsProcessor(config.text_config)
def pad_input_ids(
self, input_ids: List[int], mm_inputs: MultimodalInputs
) -> List[int]:
def pad_input_ids(self, input_ids: array, mm_inputs: MultimodalInputs) -> array:
pattern = MultiModalityDataPaddingPatternMultimodalTokens()
result = pattern.pad_input_tokens(input_ids, mm_inputs)
return result
+2 -1
View File
@@ -31,6 +31,7 @@ References:
- https://huggingface.co/lightonai/LightOnOCR-2-1B
"""
from array import array
from dataclasses import fields
from typing import Iterable, List, Tuple
@@ -125,7 +126,7 @@ class LightOnOCRForConditionalGeneration(nn.Module):
quant_config=quant_config,
)
def pad_input_ids(self, input_ids: List[int], mm_inputs: MultimodalInputs):
def pad_input_ids(self, input_ids: array, mm_inputs: MultimodalInputs) -> array:
pattern = MultiModalityDataPaddingPatternMultimodalTokens()
return pattern.pad_input_tokens(input_ids, mm_inputs)
+1 -1
View File
@@ -650,7 +650,7 @@ class LlavaForConditionalGeneration(LlavaBaseForCausalLM):
def dtype(self):
return self.torch_dtype
def pad_input_ids(self, input_ids: List[int], image_inputs: MultimodalInputs):
def pad_input_ids(self, input_ids: array, image_inputs: MultimodalInputs) -> array:
if hasattr(self.vision_tower, "pad_input_ids"):
return self.vision_tower.pad_input_ids(input_ids, image_inputs)
else:
+2 -1
View File
@@ -19,6 +19,7 @@ pattern.
"""
import logging
from array import array
from typing import Any, Dict, Iterable, List, Optional, Set, Tuple
import torch
@@ -132,7 +133,7 @@ class LocateAnythingForConditionalGeneration(nn.Module):
assert isinstance(image_features, list)
return self.multi_modal_projector(torch.cat(image_features))
def pad_input_ids(self, input_ids: List[int], mm_inputs: MultimodalInputs):
def pad_input_ids(self, input_ids: array, mm_inputs: MultimodalInputs) -> array:
pattern = MultiModalityDataPaddingPatternMultimodalTokens()
return pattern.pad_input_tokens(input_ids, mm_inputs)
+2 -1
View File
@@ -1,6 +1,7 @@
import collections
import collections.abc
import logging
from array import array
from collections.abc import Callable, Sequence
from typing import Iterable, List, Optional, Tuple, TypeAlias, cast
@@ -502,7 +503,7 @@ class MiDashengLMModel(nn.Module):
self.logits_processor = self.language_model.logits_processor
self.quant_config = quant_config
def pad_input_ids(self, input_ids: List[int], mm_inputs: MultimodalInputs):
def pad_input_ids(self, input_ids: array, mm_inputs: MultimodalInputs) -> array:
"""Pad input IDs with multimodal tokens."""
pattern = MultiModalityDataPaddingPatternMultimodalTokens()
return pattern.pad_input_tokens(input_ids, mm_inputs)
+2 -1
View File
@@ -15,6 +15,7 @@
import logging
import math
import re
from array import array
from typing import Any, Dict, Iterable, List, Optional, Tuple, Union
import torch
@@ -1273,7 +1274,7 @@ class MiMoV2ForCausalLM(nn.Module, AudioEncoderMixin):
)
return self.model.get_input_embedding(input_ids)
def pad_input_ids(self, input_ids: List[int], mm_inputs: MultimodalInputs):
def pad_input_ids(self, input_ids: array, mm_inputs: MultimodalInputs) -> array:
pattern = MultiModalityDataPaddingPatternMultimodalTokens()
return pattern.pad_input_tokens(input_ids, mm_inputs)
+2 -1
View File
@@ -7,6 +7,7 @@ state_dict aligns 1:1 with ``self.named_parameters()``.
"""
import logging
from array import array
from typing import Any, Iterable, List, Optional, Tuple
import torch
@@ -56,7 +57,7 @@ class MiMoV2ASRForCausalLM(MiMoForCausalLM, AudioEncoderMixin):
super().__init__(config, quant_config=quant_config, prefix=prefix)
self.build_audio_encoder(MiMoAudioEncoderConfig(**config.audio_config))
def pad_input_ids(self, input_ids: List[int], mm_inputs: MultimodalInputs):
def pad_input_ids(self, input_ids: array, mm_inputs: MultimodalInputs) -> array:
pattern = MultiModalityDataPaddingPatternMultimodalTokens()
return pattern.pad_input_tokens(input_ids, mm_inputs)
+2 -1
View File
@@ -16,6 +16,7 @@
"""Inference-only MiniCPM-o model compatible with HuggingFace weights."""
import math
from array import array
from dataclasses import dataclass
from typing import Any, Iterable, List, Literal, Optional, Tuple, Union
@@ -1518,7 +1519,7 @@ class MiniCPMO(MiniCPMBaseModel):
return resampler.to(device=get_device(), dtype=torch.get_default_dtype())
def pad_input_ids(self, input_ids: List[int], mm_input: MultimodalInputs):
def pad_input_ids(self, input_ids: array, mm_input: MultimodalInputs) -> array:
# Get all special token IDs
im_start_id: int = mm_input.im_start_id
im_end_id: int = mm_input.im_end_id
+5 -4
View File
@@ -22,6 +22,7 @@
"""Inference-only MiniCPM-V model compatible with HuggingFace weights."""
import types
from array import array
from functools import partial
from itertools import chain
from typing import (
@@ -983,7 +984,7 @@ class MiniCPMV2_6(MiniCPMBaseModel):
)
return self.resampler(vision_embedding, tgt_sizes)
def pad_input_ids(self, input_ids: List[int], image_inputs: MultimodalInputs):
def pad_input_ids(self, input_ids: array, image_inputs: MultimodalInputs) -> array:
# Get all special token IDs
im_start_id: int = image_inputs.im_start_id
im_end_id: int = image_inputs.im_end_id
@@ -1149,7 +1150,7 @@ class MiniCPMV4_0(MiniCPMBaseModel):
)
return self.resampler(vision_embedding, tgt_sizes)
def pad_input_ids(self, input_ids: List[int], image_inputs: MultimodalInputs):
def pad_input_ids(self, input_ids: array, image_inputs: MultimodalInputs) -> array:
# Get all special token IDs
im_start_id: int = image_inputs.im_start_id
im_end_id: int = image_inputs.im_end_id
@@ -1319,7 +1320,7 @@ class MiniCPMV4_5(MiniCPMBaseModel):
)
return self.resampler(vision_embedding, tgt_sizes)
def pad_input_ids(self, input_ids: List[int], image_inputs: MultimodalInputs):
def pad_input_ids(self, input_ids: array, image_inputs: MultimodalInputs) -> array:
# Get all special token IDs
im_start_id: int = image_inputs.im_start_id
im_end_id: int = image_inputs.im_end_id
@@ -1544,7 +1545,7 @@ class MiniCPMV4_6(MiniCPMBaseModel):
# source. sglang's dispatcher routes by ``get_{modality}_feature``.
get_video_feature = get_image_feature
def pad_input_ids(self, input_ids: List[int], image_inputs: MultimodalInputs):
def pad_input_ids(self, input_ids: array, image_inputs: MultimodalInputs) -> array:
im_start_id: int = image_inputs.im_start_id
im_end_id: int = image_inputs.im_end_id
slice_start_id: int = image_inputs.slice_start_id
+2 -1
View File
@@ -1,6 +1,7 @@
# SPDX-License-Identifier: Apache-2.0
import logging
from array import array
from typing import Iterable, List, Optional, Tuple
import torch
@@ -195,7 +196,7 @@ class MiniMaxM3SparseForConditionalGeneration(nn.Module):
text_config
)
def pad_input_ids(self, input_ids: List[int], mm_inputs: MultimodalInputs):
def pad_input_ids(self, input_ids: array, mm_inputs: MultimodalInputs) -> array:
return MultiModalityDataPaddingPatternMultimodalTokens().pad_input_tokens(
input_ids, mm_inputs
)
+2 -1
View File
@@ -3,6 +3,7 @@ import logging
import math
import os
import re
from array import array
from collections.abc import Iterable
from typing import List, Optional, Set, Tuple
@@ -559,7 +560,7 @@ class Llama4ForConditionalGeneration(nn.Module):
except (OSError, json_lib.JSONDecodeError, KeyError):
return False
def pad_input_ids(self, input_ids: List[int], mm_inputs: MultimodalInputs):
def pad_input_ids(self, input_ids: array, mm_inputs: MultimodalInputs) -> array:
return self.padding_pattern.pad_input_tokens(input_ids, mm_inputs)
def get_image_feature(
+2 -1
View File
@@ -13,6 +13,7 @@
# ==============================================================================
import logging
from array import array
from typing import Iterable, List, Optional, Tuple
import torch
@@ -977,7 +978,7 @@ class MuseGlimmerForConditionalGeneration(MuseGlimmerForCausalLM):
config.hidden_size, eps=config.rms_norm_eps, has_weight=False
)
def pad_input_ids(self, input_ids: List[int], mm_inputs: MultimodalInputs):
def pad_input_ids(self, input_ids: array, mm_inputs: MultimodalInputs) -> array:
return MultiModalityDataPaddingPatternMultimodalTokens().pad_input_tokens(
input_ids, mm_inputs
)
+2 -1
View File
@@ -16,6 +16,7 @@
# Adapted from https://github.com/vllm-project/vllm/blob/main/vllm/model_executor/models/nano_nemotron_vl.py
import logging
from array import array
from collections import deque
from typing import Iterable
@@ -116,7 +117,7 @@ class NemotronH_Nano_VL_V2(EVS):
self.config = config
def pad_input_ids(self, input_ids: list[int], mm_inputs: MultimodalInputs):
def pad_input_ids(self, input_ids: array, mm_inputs: MultimodalInputs) -> array:
im_start_id: int = mm_inputs.im_start_id
im_end_id: int = mm_inputs.im_end_id
+2 -3
View File
@@ -1,5 +1,6 @@
import itertools
import math
from array import array
from collections.abc import Iterable
from typing import Any
@@ -218,9 +219,7 @@ class NVILAForConditionalGeneration(nn.Module):
)
weight_loader(param, loaded_weight)
def pad_input_ids(
self, input_ids: list[int], mm_inputs: MultimodalInputs
) -> list[int]:
def pad_input_ids(self, input_ids: array, mm_inputs: MultimodalInputs) -> array:
pattern = MultiModalityDataPaddingPatternMultimodalTokens()
return pattern.pad_input_tokens(input_ids, mm_inputs)
+2 -3
View File
@@ -1,4 +1,5 @@
import math
from array import array
from collections.abc import Iterable
from typing import Any
@@ -178,9 +179,7 @@ class NVILALiteForConditionalGeneration(nn.Module):
)
weight_loader(param, loaded_weight)
def pad_input_ids(
self, input_ids: list[int], mm_inputs: MultimodalInputs
) -> list[int]:
def pad_input_ids(self, input_ids: array, mm_inputs: MultimodalInputs) -> array:
pattern = MultiModalityDataPaddingPatternMultimodalTokens()
return pattern.pad_input_tokens(input_ids, mm_inputs)
+2 -1
View File
@@ -24,6 +24,7 @@ projections run once for the batch instead of once per image.
"""
import itertools
from array import array
from collections.abc import Iterable
from typing import List, Optional, Set, Tuple
@@ -584,7 +585,7 @@ class PaddleOCRVLForConditionalGeneration(Ernie4_5_ForCausalLM):
)
self.is_mrope_enabled = "mrope_section" in (self.config.rope_scaling or {})
def pad_input_ids(self, input_ids: List[int], mm_inputs: MultimodalInputs):
def pad_input_ids(self, input_ids: array, mm_inputs: MultimodalInputs) -> array:
pattern = MultiModalityDataPaddingPatternMultimodalTokens()
return pattern.pad_input_tokens(input_ids, mm_inputs)
+2 -1
View File
@@ -20,6 +20,7 @@
import logging
import math
import re
from array import array
from collections.abc import Iterable
from typing import List, Optional, Tuple
@@ -475,7 +476,7 @@ class Phi4MMForCausalLM(nn.Module):
return hidden_states
def pad_input_ids(self, input_ids: List[int], mm_inputs: MultimodalInputs):
def pad_input_ids(self, input_ids: array, mm_inputs: MultimodalInputs) -> array:
pattern = MultiModalityDataPaddingPatternMultimodalTokens()
return pattern.pad_input_tokens(input_ids, mm_inputs)
+3 -2
View File
@@ -16,6 +16,7 @@
Using mistral-community/pixtral-12b as reference.
"""
from array import array
from dataclasses import dataclass, fields
from typing import Iterable, List, Optional, Set, Tuple, Union
@@ -137,7 +138,7 @@ class PixtralForConditionalGeneration(nn.Module):
self.vision_args, dim=self.config.text_config.hidden_size
)
def pad_input_ids(self, input_ids: List[int], mm_inputs: MultimodalInputs):
def pad_input_ids(self, input_ids: array, mm_inputs: MultimodalInputs) -> array:
pattern = MultiModalityDataPaddingPatternMultimodalTokens()
return pattern.pad_input_tokens(input_ids, mm_inputs)
@@ -858,7 +859,7 @@ class PixtralHFVisionModel(nn.Module):
DEFAULT_IMAGE_TOKEN_ID = 10
def pad_input_ids(self, input_ids: List[int], mm_inputs: MultimodalInputs):
def pad_input_ids(self, input_ids: array, mm_inputs: MultimodalInputs) -> array:
return self.input_padder.pad_input_tokens(input_ids, mm_inputs)
def __init__(
+2 -1
View File
@@ -1,4 +1,5 @@
import copy
from array import array
from typing import Iterable, List, Optional, Set, Tuple
import torch
@@ -101,7 +102,7 @@ class POINTSV15ChatModel(nn.Module):
prefix=add_prefix("vision_projector", prefix),
)
def pad_input_ids(self, input_ids: List[int], mm_inputs: MultimodalInputs):
def pad_input_ids(self, input_ids: array, mm_inputs: MultimodalInputs) -> array:
pattern = MultiModalityDataPaddingPatternMultimodalTokens()
return pattern.pad_input_tokens(input_ids, mm_inputs)
+2 -1
View File
@@ -25,6 +25,7 @@
import logging
import re
from array import array
from functools import partial
from typing import Iterable, List, Optional, Tuple, Type
@@ -767,7 +768,7 @@ class Qwen2_5_VLForConditionalGeneration(nn.Module):
# For EAGLE3 support
self.capture_aux_hidden_states = False
def pad_input_ids(self, input_ids: List[int], mm_inputs: MultimodalInputs):
def pad_input_ids(self, input_ids: array, mm_inputs: MultimodalInputs) -> array:
pattern = MultiModalityDataPaddingPatternMultimodalTokens()
return pattern.pad_input_tokens(input_ids, mm_inputs)
+2 -1
View File
@@ -24,6 +24,7 @@
"""Inference-only Qwen2-Audio model compatible with HuggingFace weights."""
import logging
from array import array
from typing import Any, Iterable, List, Optional, Tuple
import torch
@@ -100,7 +101,7 @@ class Qwen2AudioForConditionalGeneration(nn.Module):
)
self.pattern = MultiModalityDataPaddingPatternMultimodalTokens()
def pad_input_ids(self, input_ids: List[int], mm_inputs: MultimodalInputs):
def pad_input_ids(self, input_ids: array, mm_inputs: MultimodalInputs) -> array:
return self.pattern.pad_input_tokens(input_ids, mm_inputs)
def get_audio_feature(self, items: List[MultimodalDataItem]) -> torch.Tensor:
+2 -1
View File
@@ -24,6 +24,7 @@
"""Inference-only Qwen2-VL model compatible with HuggingFace weights."""
import logging
from array import array
from functools import lru_cache, partial
from typing import Iterable, List, Optional, Tuple, Type, TypedDict
@@ -492,7 +493,7 @@ class Qwen2VLForConditionalGeneration(nn.Module):
self.logits_processor = LogitsProcessor(config)
self.pooler = Pooler(pooling_type=PoolingType.LAST, normalize=True)
def pad_input_ids(self, input_ids: List[int], mm_inputs: MultimodalInputs):
def pad_input_ids(self, input_ids: array, mm_inputs: MultimodalInputs) -> array:
pattern = MultiModalityDataPaddingPatternMultimodalTokens()
return pattern.pad_input_tokens(input_ids, mm_inputs)
+2 -1
View File
@@ -1,6 +1,7 @@
"""Qwen3-ASR model compatible with HuggingFace weights"""
import logging
from array import array
from typing import Any, Iterable, List, Optional, Tuple
import torch
@@ -66,7 +67,7 @@ class Qwen3ASRForConditionalGeneration(nn.Module):
)
self.pattern = MultiModalityDataPaddingPatternMultimodalTokens()
def pad_input_ids(self, input_ids: List[int], mm_inputs: MultimodalInputs):
def pad_input_ids(self, input_ids: array, mm_inputs: MultimodalInputs) -> array:
return self.pattern.pad_input_tokens(input_ids, mm_inputs)
def get_audio_feature(self, items: List[MultimodalDataItem]) -> torch.Tensor:
+2 -1
View File
@@ -16,6 +16,7 @@
import logging
import re
from array import array
from collections import defaultdict
from functools import lru_cache, partial
from typing import Callable, Iterable, List, Optional, Tuple, Union
@@ -1420,7 +1421,7 @@ class Qwen3VLForConditionalGeneration(nn.Module):
cfg = getattr(model, "config", None)
return int(getattr(cfg, "num_hidden_layers", 0))
def pad_input_ids(self, input_ids: List[int], mm_inputs: MultimodalInputs):
def pad_input_ids(self, input_ids: array, mm_inputs: MultimodalInputs) -> array:
if mm_inputs and mm_inputs.mm_items:
_require_vision(self)
pattern = MultiModalityDataPaddingPatternMultimodalTokens()
@@ -14,6 +14,7 @@
"""Inference-only Sarashina2Vision model compatible with HuggingFace weights."""
import logging
from array import array
from typing import Iterable, List, Optional, Tuple
import torch
@@ -107,7 +108,7 @@ class Sarashina2VisionForCausalLM(nn.Module):
self.logits_processor = LogitsProcessor(config)
self.pooler = Pooler(pooling_type=PoolingType.LAST, normalize=True)
def pad_input_ids(self, input_ids: List[int], mm_inputs: MultimodalInputs):
def pad_input_ids(self, input_ids: array, mm_inputs: MultimodalInputs) -> array:
"""Pad input tokens with multimodal data hashes for RadixAttention."""
pattern = MultiModalityDataPaddingPatternMultimodalTokens()
return pattern.pad_input_tokens(input_ids, mm_inputs)
+2 -1
View File
@@ -1,5 +1,6 @@
import logging
import math
from array import array
from math import sqrt
from typing import Any, Dict, Iterable, List, Optional, Tuple
@@ -895,7 +896,7 @@ class Step3VLForConditionalGeneration(nn.Module):
)
return self._flatten_embeddings(merged_image_features)
def pad_input_ids(self, input_ids: List[int], mm_inputs: MultimodalInputs):
def pad_input_ids(self, input_ids: array, mm_inputs: MultimodalInputs) -> array:
pattern = MultiModalityDataPaddingPatternMultimodalTokens()
return pattern.pad_input_tokens(input_ids, mm_inputs)
+2 -1
View File
@@ -2,6 +2,7 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""This is basically a copy from perception_models/core/vision_encoder/pe.py"""
from array import array
from functools import partial
from typing import Callable, Iterable, List, Optional, Tuple
@@ -570,7 +571,7 @@ class StepVLForConditionalGeneration(nn.Module):
)
return self._flatten_embeddings(merged_image_features)
def pad_input_ids(self, input_ids: List[int], mm_inputs: MultimodalInputs):
def pad_input_ids(self, input_ids: array, mm_inputs: MultimodalInputs) -> array:
pattern = MultiModalityDataPaddingPatternMultimodalTokens()
return pattern.pad_input_tokens(input_ids, mm_inputs)
+2 -1
View File
@@ -1,3 +1,4 @@
from array import array
from typing import Iterable, List, Optional, Tuple
import torch
@@ -128,7 +129,7 @@ class Step3p7ForConditionalGeneration(nn.Module):
)
return self._flatten_embeddings(merged_image_features)
def pad_input_ids(self, input_ids: List[int], mm_inputs: MultimodalInputs):
def pad_input_ids(self, input_ids: array, mm_inputs: MultimodalInputs) -> array:
pattern = MultiModalityDataPaddingPatternMultimodalTokens()
return pattern.pad_input_tokens(input_ids, mm_inputs)
+2 -1
View File
@@ -21,6 +21,7 @@
import inspect
import logging
import re
from array import array
from collections.abc import Iterable, Mapping
from contextlib import contextmanager
from typing import List, Literal, Optional, Tuple, Union
@@ -1368,7 +1369,7 @@ class MultiModalMixin:
rope_type = str(getattr(self.text_config, "rope_type", "")).lower()
return "mrope" in rope_type
def pad_input_ids(self, input_ids: list[int], mm_inputs: MultimodalInputs):
def pad_input_ids(self, input_ids: array, mm_inputs: MultimodalInputs) -> array:
return input_ids
def _get_modality_encoder(self, modality_name: str):
+2 -1
View File
@@ -1,6 +1,7 @@
"""Standalone UNLIMITED-OCR model (SAM + CLIP vision encoders, Deepseek backbone)."""
import logging
from array import array
from typing import Iterable, List, Optional, Set, Tuple, TypeAlias, Union
import torch
@@ -311,7 +312,7 @@ class UnlimitedOCRForCausalLM(nn.Module):
)
return inputs_embeds
def pad_input_ids(self, input_ids: List[int], mm_inputs: MultimodalInputs):
def pad_input_ids(self, input_ids: array, mm_inputs: MultimodalInputs) -> array:
"""Pad input token IDs with multimodal placeholder tokens."""
pattern = MultiModalityDataPaddingPatternMultimodalTokens()
return pattern.pad_input_tokens(input_ids, mm_inputs)
+2 -1
View File
@@ -9,6 +9,7 @@
"""Inference-only Voxtral (speech-to-text) model."""
import math
from array import array
from typing import Any, Iterable, List, Optional, Tuple
import torch
@@ -337,7 +338,7 @@ class VoxtralForConditionalGeneration(nn.Module):
return results
def pad_input_ids(self, input_ids: List[int], mm_inputs: MultimodalInputs):
def pad_input_ids(self, input_ids: array, mm_inputs: MultimodalInputs) -> array:
return self.pattern.pad_input_tokens(input_ids, mm_inputs)
def get_audio_feature(self, items: List[MultimodalDataItem]) -> torch.Tensor:
+6 -4
View File
@@ -15,6 +15,8 @@
# ==============================================================================
# Adapted from https://github.com/vllm-project/vllm/blob/main/vllm/multimodal/evs.py
from array import array
import torch
@@ -122,8 +124,7 @@ def tokens_per_frame(
def replace_offsets_with_tokens_per_frame(
*,
pre_chunked_input_ids: list[int],
pre_chunked_input_ids: list[int] | array,
num_tokens_per_frame: list[int],
frame_offsets_inclusive: list[tuple[int, int]],
filler_token_id: int,
@@ -157,12 +158,13 @@ def replace_offsets_with_tokens_per_frame(
... filler_token_id=0,
... ) == [0, 1, 4, 0, 0, 0, 0, 5, 9, 10]
"""
assert isinstance(pre_chunked_input_ids, list)
# Encoder metadata uses lists; model padding can replace it with packed tokens.
assert isinstance(pre_chunked_input_ids, (list, array))
ids = pre_chunked_input_ids
if len(frame_offsets_inclusive) == 1:
"""There might be no frame separators, in which case there will be one contiguous span of tokens"""
final = ids[0 : frame_offsets_inclusive[0][0]]
final = list(ids[0 : frame_offsets_inclusive[0][0]])
frames = [filler_token_id] * sum(num_tokens_per_frame)
final.extend(frames)
else:
+7
View File
@@ -1281,6 +1281,13 @@ class Range(NamedTuple):
return self.end - self.start
def assert_int64_array(values: array, name: str) -> None:
"""Require a signed int64 array suitable for zero-copy tensor views."""
assert (
isinstance(values, array) and values.typecode == "q" and values.itemsize == 8
), f"{name} must be array('q') with 8-byte items"
def flatten_arrays_to_pinned_cpu(parts: List[array[int]], pin: bool) -> torch.Tensor:
"""Flatten array.array('q') buffers into one int64 CPU tensor.