[MM] Keep scheduler padding in packed token arrays (#40357)

This commit is contained in:
Jialin Ouyang
2026-09-22 15:12:07 -07:00
committed by GitHub
parent c19dc43cc7
commit b77833c504
64 changed files with 376 additions and 110 deletions
+198
View File
@@ -0,0 +1,198 @@
"""CPU comparison of baseline and optimized offset-based scheduler padding.
Run from the repository root:
PYTHONPATH=python python benchmark/io/bench_mm_token_padding.py --baseline-ref BASE_SHA
Inputs are synthetic. Both paths include the scheduler's final ownership copy.
This measures padding CPU time, not model forward time or end-to-end TTFT.
"""
import argparse
import ast
import gc
import json
import platform
import statistics
import subprocess
import time
from array import array
from pathlib import Path
import torch
from sglang.srt.managers import mm_utils
from sglang.srt.managers.mm_utils import (
MultiModalityDataPaddingPatternMultimodalTokens,
duplicate_pad_mm_input_ids,
)
from sglang.srt.managers.schedule_batch import (
Modality,
MultimodalDataItem,
MultimodalInputs,
)
def load_baseline(ref):
"""Load the exact baseline helper classes without replacing installed SGLang."""
root = Path(__file__).resolve().parents[2]
commit = subprocess.check_output(
["git", "rev-parse", "--verify", f"{ref}^{{commit}}"], cwd=root, text=True
).strip()
path = "python/sglang/srt/managers/mm_utils.py"
source = subprocess.check_output(
["git", "show", f"{commit}:{path}"], cwd=root, text=True
)
names = {
"MultiModalityDataPaddingPattern",
"MultiModalityDataPaddingPatternMultimodalTokens",
}
classes = [
node
for node in ast.parse(source).body
if isinstance(node, ast.ClassDef) and node.name in names
]
if {node.name for node in classes} != names:
raise ValueError("Baseline does not contain the expected padding helpers")
namespace = dict(vars(mm_utils))
# Reuse dependency imports, but compile the original method bodies verbatim.
exec(
compile(ast.Module(body=classes, type_ignores=[]), f"{commit}:{path}", "exec"),
namespace,
)
return commit, namespace["MultiModalityDataPaddingPatternMultimodalTokens"]()
def make_case(num_tokens, num_items, span_tokens):
if num_items and num_tokens < 3 * num_items:
raise ValueError(
"Each item needs three tokens for its span and surrounding text"
)
input_ids = array("q", range(257, 257 + num_tokens))
items = []
stride = num_tokens // max(num_items, 1)
for index in range(num_items):
start = index * stride + 1
end = start + min(span_tokens, stride - 2) - 1
items.append(
MultimodalDataItem(
modality=Modality.IMAGE,
pad_value=2**40 + index,
offsets=[(start, end)],
)
)
return input_ids, MultimodalInputs(mm_items=items, im_token_id=1)
def benchmark_case(num_tokens, num_items, args, baseline):
pattern = MultiModalityDataPaddingPatternMultimodalTokens()
input_ids, mm_inputs = make_case(num_tokens, num_items, args.span_tokens)
original = input_ids.tobytes()
def baseline_padding():
return array("q", baseline.pad_input_tokens(input_ids, mm_inputs))
def optimized_padding():
return duplicate_pad_mm_input_ids(
input_ids, mm_inputs, pattern.pad_input_tokens
)
expected = baseline_padding()
actual = optimized_padding()
assert actual == expected, "Token values differ"
assert input_ids.tobytes() == original, "Original input was mutated"
assert actual is not input_ids, "Scheduler result aliases the original input"
functions = {"baseline": baseline_padding, "optimized": optimized_padding}
timings = {name: [] for name in functions}
for _ in range(args.warmups):
for function in functions.values():
function()
gc_was_enabled = gc.isenabled()
gc.disable()
try:
for trial in range(args.trials):
order = (
("baseline", "optimized") if trial % 2 else ("optimized", "baseline")
)
for name in order:
start = time.perf_counter_ns()
result = functions[name]()
elapsed = time.perf_counter_ns() - start
timings[name].append(elapsed / 1e6)
del result
finally:
if gc_was_enabled:
gc.enable()
assert input_ids.tobytes() == original
medians = {name: statistics.median(samples) for name, samples in timings.items()}
return {
"tokens": num_tokens,
"items": num_items,
"baseline_ms": medians["baseline"],
"optimized_ms": medians["optimized"],
"speedup": medians["baseline"] / medians["optimized"],
}
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument(
"--baseline-ref", required=True, help="Local pre-change git ref"
)
parser.add_argument(
"--tokens", nargs="+", type=int, default=[128, 4096, 16384, 131072]
)
parser.add_argument("--items", nargs="+", type=int, default=[0, 1, 10])
parser.add_argument("--span-tokens", type=int, default=32)
parser.add_argument("--trials", type=int, default=41)
parser.add_argument("--warmups", type=int, default=5)
parser.add_argument("--threads", type=int, default=1)
parser.add_argument("--json", action="store_true")
args = parser.parse_args()
if (
min(args.tokens + args.items) < 0
or min(args.span_tokens, args.trials, args.threads) < 1
or args.warmups < 0
):
parser.error("Lengths/items/warmups must be nonnegative; other counts positive")
if any(n < 3 * items for n in args.tokens for items in args.items):
parser.error("Each token length must accommodate at least 3 tokens per item")
torch.set_num_threads(args.threads)
baseline_commit, baseline = load_baseline(args.baseline_ref)
rows = [
benchmark_case(tokens, items, args, baseline)
for tokens in args.tokens
for items in args.items
]
if args.json:
print(
json.dumps(
{
"baseline_commit": baseline_commit,
"python": platform.python_version(),
"torch": torch.__version__,
"architecture": platform.machine(),
"threads": args.threads,
"trials": args.trials,
"warmups": args.warmups,
"span_tokens": args.span_tokens,
"results": rows,
},
indent=2,
)
)
return
print("Offset-based CPU padding; median milliseconds; synthetic inputs")
print(
f"{'Tokens':>8} {'Items':>6} {'Baseline (ms)':>14} {'Optimized (ms)':>14} {'Speedup':>9}"
)
for row in rows:
print(
f"{row['tokens']:>8} {row['items']:>6} "
f"{row['baseline_ms']:>14.4f} {row['optimized_ms']:>14.4f} "
f"{row['speedup']:>8.2f}x"
)
if __name__ == "__main__":
main()
+35 -17
View File
@@ -9,9 +9,10 @@ import os
import pickle
import sys
from abc import abstractmethod
from array import array
from collections import defaultdict
from multiprocessing import shared_memory
from typing import Any, Dict, List, Optional, Tuple
from typing import Any, Callable, Dict, List, Optional, Tuple
import numpy as np
import torch
@@ -49,7 +50,11 @@ from sglang.srt.runtime_context import (
get_server_args,
get_serving,
)
from sglang.srt.utils import flatten_nested_list, print_warning_once
from sglang.srt.utils import (
assert_int64_array,
flatten_nested_list,
print_warning_once,
)
from sglang.srt.utils.stale_shm_cleanup import make_shm_name
from sglang.utils import logger
@@ -232,6 +237,19 @@ class TransportProxyTensor(torch.Tensor):
return self._metadata.get("transport_mode", "default")
def duplicate_pad_mm_input_ids(
input_ids: array,
mm_inputs: MultimodalInputs,
pad_input_ids_func: Callable[[array, MultimodalInputs], array],
) -> array:
"""Validate and duplicate model-padded tokens for scheduler ownership."""
padded_ids = pad_input_ids_func(input_ids, mm_inputs)
assert_int64_array(padded_ids, "pad_input_ids result")
# Preserve scheduler ownership: no-op models return the unpadded input,
# and other models can retain the result in multimodal metadata.
return padded_ids[:]
class MultiModalityDataPaddingPattern:
"""
Data tokens (like image tokens) often need special handling during padding
@@ -240,11 +258,10 @@ class MultiModalityDataPaddingPattern:
"""
@abstractmethod
def pad_input_tokens(
self, input_ids: List[int], mm_inputs: MultimodalInputs
) -> List[int]:
def pad_input_tokens(self, input_ids: array, mm_inputs: MultimodalInputs) -> array:
"""
Pad the input ids sequence containing data tokens, and replace them with pad_values
Replace data tokens with pad_values using native int64 arrays.
Implementations must not mutate the original input.
"""
pass
@@ -273,12 +290,11 @@ class MultiModalityDataPaddingPatternTokenPairs(MultiModalityDataPaddingPattern)
s for s, _e in data_token_pairs
]
def pad_input_tokens(
self, input_ids: List[int], mm_inputs: MultimodalInputs
) -> List[int]:
def pad_input_tokens(self, input_ids: array, mm_inputs: MultimodalInputs) -> array:
"""
This function will replace the data-tokens in between with pad_values accordingly
"""
assert_int64_array(input_ids, "input_ids")
pad_values = [item.pad_value for item in mm_inputs.mm_items]
data_token_pairs = self.data_token_id_pairs
mm_inputs.data_offsets = []
@@ -292,7 +308,7 @@ class MultiModalityDataPaddingPatternTokenPairs(MultiModalityDataPaddingPattern)
start_token_ids = {s for s, _e in data_token_pairs}
end_tokens_ids = {e for _s, e in data_token_pairs}
padded_ids = []
padded_ids = array("q")
last_idx = 0
data_idx = -1
@@ -314,7 +330,8 @@ class MultiModalityDataPaddingPatternTokenPairs(MultiModalityDataPaddingPattern)
num_tokens = end_idx - start_idx - 1
pad_value = pad_values[data_idx]
padded_ids.extend([pad_value] * num_tokens)
if num_tokens > 0:
padded_ids.extend(array("q", [pad_value]) * num_tokens)
last_idx = end_idx
@@ -329,17 +346,19 @@ class MultiModalityDataPaddingPatternMultimodalTokens(MultiModalityDataPaddingPa
e.g. <image><image>....<image>, or <audio><audio>...<audio>
"""
def pad_input_tokens(
self, input_ids: List[int], mm_inputs: MultimodalInputs
) -> List[int]:
def pad_input_tokens(self, input_ids: array, mm_inputs: MultimodalInputs) -> array:
"""
Replaces multimodal tokens in input_ids with corresponding pad_values from mm_items.
Each modality (image, audio, video) is handled separately based on its token_id.
"""
assert_int64_array(input_ids, "input_ids")
if not input_ids or not mm_inputs.mm_items:
return input_ids
input_ids_tensor = torch.as_tensor(input_ids)
# as_tensor(array) walks and boxes every token. View a bulk copy instead,
# keeping the original request's unpadded token buffer unchanged.
padded_input_ids = input_ids[:]
input_ids_tensor = torch.frombuffer(padded_input_ids, dtype=torch.int64)
# Replace multimodal tokens using per-item offsets
items_by_modality = defaultdict(list)
@@ -362,8 +381,7 @@ class MultiModalityDataPaddingPatternMultimodalTokens(MultiModalityDataPaddingPa
for offset in items[i].offsets:
input_ids_tensor[offset[0] : offset[1] + 1] = item.pad_value
ret_input_ids = input_ids_tensor.tolist()
return ret_input_ids
return padded_input_ids
# masked_scatter_ materializes the expanded [num_tokens, hidden] bool mask plus
+5 -6
View File
@@ -193,6 +193,7 @@ from sglang.srt.managers.min_free_slots_delayer import (
MinFreeSlotsDelayer,
resolve_min_free_slots,
)
from sglang.srt.managers.mm_utils import duplicate_pad_mm_input_ids
from sglang.srt.managers.multimodal_processor import get_mm_processor, import_processors
from sglang.srt.managers.overlap_utils import (
RelayPayload,
@@ -1214,7 +1215,6 @@ class Scheduler(
)
self.dp_tp_cpu_group = self.dp_tp_group.cpu_group
# TODO(Jialin): Migrate pad_input_ids implementations to return array.
self.pad_input_ids_func = self.tp_worker.get_pad_input_ids_func()
set_random_seed(self.random_seed)
@@ -2994,8 +2994,8 @@ class Scheduler(
not self._try_apply_padded_mm_input_ids(recv_req, req, image_inputs)
and self.pad_input_ids_func
):
req.origin_input_ids = array(
"q", self.pad_input_ids_func(req.origin_input_ids, image_inputs)
req.origin_input_ids = duplicate_pad_mm_input_ids(
req.origin_input_ids, image_inputs, self.pad_input_ids_func
)
req.extend_image_inputs(image_inputs)
self._maybe_compute_mrope_positions(req)
@@ -3468,9 +3468,8 @@ class Scheduler(
not self._try_apply_padded_mm_input_ids(recv_req, req, image_inputs)
and self.pad_input_ids_func
):
# See companion call site above for the array.array wrap rationale.
req.origin_input_ids = array(
"q", self.pad_input_ids_func(req.origin_input_ids, image_inputs)
req.origin_input_ids = duplicate_pad_mm_input_ids(
req.origin_input_ids, image_inputs, self.pad_input_ids_func
)
req.extend_image_inputs(image_inputs)
@@ -55,6 +55,7 @@ from sglang.srt.mem_cache.unified_cache.unified_tree_core_interface import (
UnifiedTreeCoreInterface,
)
from sglang.srt.runtime_context import get_exec, mamba_cache_chunk_size
from sglang.srt.utils import assert_int64_array
if TYPE_CHECKING:
from sglang.srt.managers.schedule_batch import Req
@@ -72,9 +73,7 @@ def _radix_key_buffer(key: RadixKey) -> array:
"""The key's token ids honoring `limit`; view-independent since the
binding derives its own atoms."""
token_ids = key.raw_token_ids()
assert isinstance(token_ids, array) and token_ids.typecode == "q", (
f"tree keys must carry array('q') token ids, got {type(token_ids).__name__}"
)
assert_int64_array(token_ids, "tree key token ids")
return token_ids
+2 -1
View File
@@ -14,6 +14,7 @@
"""Legacy Bailing multimodal wrappers for image and video inference."""
import logging
from array import array
from typing import Iterable, List, Optional, Set, Tuple
import torch
@@ -122,7 +123,7 @@ class BailingMMNativeForConditionalGeneration(nn.Module):
def get_input_embeddings(self):
return self.model.get_input_embeddings()
def pad_input_ids(self, input_ids: List[int], mm_inputs: MultimodalInputs):
def pad_input_ids(self, input_ids: array, mm_inputs: MultimodalInputs) -> array:
return self.pattern.pad_input_tokens(input_ids, mm_inputs)
def _get_vision_feature(
+2 -1
View File
@@ -14,6 +14,7 @@
"""SGLang implementation of Bailing/Ling 3 VL image and video inference."""
import logging
from array import array
from typing import Iterable, List, Optional, Set, Tuple
import torch
@@ -128,7 +129,7 @@ class BailingMoeV3VLForConditionalGeneration(nn.Module):
def get_input_embeddings(self):
return self.model.get_input_embeddings()
def pad_input_ids(self, input_ids: List[int], mm_inputs: MultimodalInputs):
def pad_input_ids(self, input_ids: array, mm_inputs: MultimodalInputs) -> array:
return self.pattern.pad_input_tokens(input_ids, mm_inputs)
def _materialize_items(self, items: List[MultimodalDataItem]) -> torch.Tensor:
+2 -1
View File
@@ -1,6 +1,7 @@
# Adapted from
# https://github.com/huggingface/transformers/blob/af9b2eaa54c150741f298d6db939af6328e1dc38/src/transformers/models/clip/modeling_clip.py
from array import array
from functools import partial
from typing import Iterable, List, Optional, Tuple, Type, Union
@@ -580,7 +581,7 @@ class CLIPModel(nn.Module):
embeddings=self.text_projection(pooled_output.embeddings)
)
def pad_input_ids(self, input_ids: List[int], image_inputs: MultimodalInputs):
def pad_input_ids(self, input_ids: array, image_inputs: MultimodalInputs) -> array:
# Clip embeddings models handle text/image separately, so we don't need to pad input ids
return input_ids
+2 -3
View File
@@ -5,6 +5,7 @@
"""Inference-only Cohere2Vision (Command-A-Vision) multimodal model."""
import math
from array import array
from typing import Iterable, List, Optional, Tuple
import torch
@@ -151,9 +152,7 @@ class Cohere2VisionForConditionalGeneration(nn.Module):
# walks ``model.model.layers``) can locate the transformer layers.
self.model = self.language_model.model
def pad_input_ids(
self, input_ids: List[int], mm_inputs: MultimodalInputs
) -> List[int]:
def pad_input_ids(self, input_ids: array, mm_inputs: MultimodalInputs) -> array:
pattern = MultiModalityDataPaddingPatternMultimodalTokens()
return pattern.pad_input_tokens(input_ids, mm_inputs)
+2 -3
View File
@@ -19,6 +19,7 @@ Arcee causal-LM structure used by SGLang, while the vision path uses the native
SigLIP2 implementation and an Edge-specific spatial-merge projector.
"""
from array import array
from typing import Iterable, List, Optional, Tuple
import numpy as np
@@ -212,9 +213,7 @@ class Cosmos3EdgeForConditionalGeneration(ArceeForCausalLM):
prefix=add_prefix("projector", prefix),
)
def pad_input_ids(
self, input_ids: List[int], mm_inputs: MultimodalInputs
) -> List[int]:
def pad_input_ids(self, input_ids: array, mm_inputs: MultimodalInputs) -> array:
pattern = MultiModalityDataPaddingPatternMultimodalTokens()
return pattern.pad_input_tokens(input_ids, mm_inputs)
@@ -19,6 +19,7 @@
import collections
import math
import os
from array import array
from dataclasses import field
from enum import Enum
from functools import partial
@@ -1995,7 +1996,7 @@ class MultiModalityCausalLM(MultiModalityPreTrainedModel):
def prepare_gen_img_embeds(self, image_ids: torch.LongTensor):
return self.gen_aligner(self.gen_embed(image_ids))
def pad_input_ids(self, input_ids: List[int], image_inputs: MultimodalInputs):
def pad_input_ids(self, input_ids: array, image_inputs: MultimodalInputs) -> array:
im_start_id = image_inputs.im_start_id
im_end_id = image_inputs.im_end_id
media_token_pairs = [(im_start_id, im_end_id)]
+2 -1
View File
@@ -22,6 +22,7 @@
import copy
import logging
import math
from array import array
from functools import partial
from typing import Iterable, List, Optional, Set, Tuple, Type, TypeAlias, Union
@@ -1763,7 +1764,7 @@ class DeepseekOCRForCausalLM(nn.Module):
return inputs_embeds
def pad_input_ids(self, input_ids: List[int], mm_inputs: MultimodalInputs):
def pad_input_ids(self, input_ids: array, mm_inputs: MultimodalInputs) -> array:
pattern = MultiModalityDataPaddingPatternMultimodalTokens()
return pattern.pad_input_tokens(input_ids, mm_inputs)
+2 -1
View File
@@ -4,6 +4,7 @@ import concurrent.futures
import functools
import logging
import time
from array import array
from contextlib import contextmanager, nullcontext
from types import SimpleNamespace
from typing import (
@@ -4981,7 +4982,7 @@ class DeepseekV4ForCausalLM(nn.Module):
def routed_experts_weights_of_layer(self):
return self._routed_experts_weights_of_layer.value
def pad_input_ids(self, input_ids, mm_inputs):
def pad_input_ids(self, input_ids: array, mm_inputs) -> array:
return MultiModalityDataPaddingPatternMultimodalTokens().pad_input_tokens(
input_ids, mm_inputs
)
+2 -1
View File
@@ -1,3 +1,4 @@
from array import array
from typing import Iterable, List, Optional, Tuple
import torch
@@ -262,7 +263,7 @@ class DeepseekVL2ForCausalLM(nn.Module):
weights_loader = getattr(param, "weight_loader", default_weight_loader)
weights_loader(param, loaded_weight)
def pad_input_ids(self, input_ids: List[int], mm_inputs: MultimodalInputs):
def pad_input_ids(self, input_ids: array, mm_inputs: MultimodalInputs) -> array:
pattern = MultiModalityDataPaddingPatternMultimodalTokens()
return pattern.pad_input_tokens(input_ids, mm_inputs)
@@ -19,6 +19,7 @@
import concurrent.futures
import logging
import math
from array import array
from dataclasses import dataclass
from enum import IntEnum, auto
from typing import (
@@ -1921,10 +1922,10 @@ class Dots3LanguageModelForCausalLM(nn.Module):
def pad_input_ids(
self,
input_ids: List[int],
input_ids: array,
mm_inputs: MultimodalInputs,
**kwargs,
) -> List[int]:
) -> array:
token_pairs = []
if mm_inputs.im_start_id is not None and mm_inputs.im_end_id is not None:
token_pairs.append((mm_inputs.im_start_id, mm_inputs.im_end_id))
@@ -2637,7 +2638,7 @@ class DotsNoteOmniThinkerForConditionalGeneration(nn.Module):
def get_input_embeddings(self):
return self.language_model.get_input_embeddings()
def pad_input_ids(self, input_ids, mm_inputs, **kwargs):
def pad_input_ids(self, input_ids: array, mm_inputs, **kwargs) -> array:
return self.language_model.pad_input_ids(input_ids, mm_inputs, **kwargs)
def get_image_feature(self, items: List[MultimodalDataItem]) -> torch.Tensor:
+2 -1
View File
@@ -2,6 +2,7 @@
# Adapted from Qwen2.5-VL SGLang implementation
import logging
from array import array
from typing import Iterable, List, Optional, Tuple
import torch
@@ -56,7 +57,7 @@ class DotsOCRForCausalLM(nn.Module):
self.logits_processor = LogitsProcessor(config)
def pad_input_ids(self, input_ids: List[int], mm_inputs: MultimodalInputs):
def pad_input_ids(self, input_ids: array, mm_inputs: MultimodalInputs) -> array:
pattern = MultiModalityDataPaddingPatternMultimodalTokens()
return pattern.pad_input_tokens(input_ids, mm_inputs)
+2 -1
View File
@@ -17,6 +17,7 @@
# limitations under the License.
"""Inference-only Dots-VL model compatible with HuggingFace weights."""
from array import array
from typing import Iterable, List, Optional, Tuple
import torch
@@ -132,7 +133,7 @@ class DotsVLMForCausalLM(nn.Module):
def get_model_config_for_expert_location(cls, config):
return DeepseekV2ForCausalLM.get_model_config_for_expert_location(config)
def pad_input_ids(self, input_ids: List[int], mm_inputs: MultimodalInputs):
def pad_input_ids(self, input_ids: array, mm_inputs: MultimodalInputs) -> array:
"""Pad input_ids with multimodal tokens"""
# Get image token ID for padding pattern
pattern = MultiModalityDataPaddingPatternMultimodalTokens()
+2 -1
View File
@@ -13,6 +13,7 @@
"""Inference-only Ernie45-VL model compatible with HuggingFace weights."""
import logging
from array import array
from functools import lru_cache, partial
from typing import Iterable, List, Optional, Tuple, Type
@@ -588,7 +589,7 @@ class Ernie4_5_VLMoeForConditionalGeneration(nn.Module):
else:
self._visual_token_ids_tensor_cache = None
def pad_input_ids(self, input_ids: List[int], mm_inputs: MultimodalInputs):
def pad_input_ids(self, input_ids: array, mm_inputs: MultimodalInputs) -> array:
pattern = MultiModalityDataPaddingPatternMultimodalTokens()
return pattern.pad_input_tokens(input_ids, mm_inputs)
+2 -3
View File
@@ -19,6 +19,7 @@
import logging
import re
from array import array
from functools import lru_cache
from typing import Iterable, List, Optional, Set, Tuple, TypedDict
@@ -201,9 +202,7 @@ class Gemma3ForConditionalGeneration(PreTrainedModel):
self.language_model.logits_processor.logit_scale *= logit_scale
self.post_init()
def pad_input_ids(
self, input_ids: List[int], image_inputs: MultimodalInputs
) -> List[int]:
def pad_input_ids(self, input_ids: array, image_inputs: MultimodalInputs) -> array:
"""Pad input IDs with image tokens."""
# Get special token IDs
im_start_id: int = image_inputs.im_start_id
+3 -2
View File
@@ -1,5 +1,6 @@
import logging
import re
from array import array
from functools import lru_cache
from typing import Iterable, List, Optional, Set, Tuple, TypedDict, Union
@@ -243,9 +244,9 @@ class Gemma3nForConditionalGeneration(PreTrainedModel):
def pad_input_ids(
self,
input_ids: List[int],
input_ids: array,
mm_inputs: MultimodalInputs,
) -> List[int]:
) -> array:
"""Pad input IDs with image and audio tokens."""
pattern = MultiModalityDataPaddingPatternMultimodalTokens()
return pattern.pad_input_tokens(input_ids, mm_inputs)
+3 -2
View File
@@ -15,6 +15,7 @@
import logging
import re
from array import array
from functools import lru_cache
from typing import Iterable, List, Optional, Set, Tuple, TypedDict, Union
@@ -296,9 +297,9 @@ class Gemma4ForConditionalGeneration(PreTrainedModel):
def pad_input_ids(
self,
input_ids: List[int],
input_ids: array,
mm_inputs: MultimodalInputs,
) -> List[int]:
) -> array:
"""Pad input IDs with image and audio tokens."""
pattern = MultiModalityDataPaddingPatternMultimodalTokens()
return pattern.pad_input_tokens(input_ids, mm_inputs)
+2 -1
View File
@@ -18,6 +18,7 @@
"""Inference-only GLM-4.1V model compatible with HuggingFace weights."""
import logging
from array import array
from functools import lru_cache
from typing import Iterable, List, Optional, Tuple
@@ -594,7 +595,7 @@ class Glm4vForConditionalGeneration(nn.Module):
# For EAGLE3 support
self.capture_aux_hidden_states = False
def pad_input_ids(self, input_ids: List[int], mm_inputs: MultimodalInputs):
def pad_input_ids(self, input_ids: array, mm_inputs: MultimodalInputs) -> array:
pattern = MultiModalityDataPaddingPatternMultimodalTokens()
return pattern.pad_input_tokens(input_ids, mm_inputs)
+2 -1
View File
@@ -1,4 +1,5 @@
import logging
from array import array
from contextlib import nullcontext
from functools import partial
from typing import Any, Dict, Iterable, List, Optional, Tuple, Union
@@ -1419,7 +1420,7 @@ class Glm5NextForConditionalGeneration(nn.Module):
# Capturing before layer k + 1 gives the completed output of layer k.
self.model.layers_to_capture = [val + 1 for val in layer_ids]
def pad_input_ids(self, input_ids: List[int], mm_inputs: MultimodalInputs):
def pad_input_ids(self, input_ids: array, mm_inputs: MultimodalInputs) -> array:
pattern = MultiModalityDataPaddingPatternMultimodalTokens()
return pattern.pad_input_tokens(input_ids, mm_inputs)
+2 -1
View File
@@ -20,6 +20,7 @@
import copy
import logging
from array import array
from typing import Any, Dict, Iterable, List, Optional, Tuple
import torch
@@ -1068,7 +1069,7 @@ class GlmImageForConditionalGeneration(nn.Module):
self.logits_processor = LogitsProcessor(logits_config)
def pad_input_ids(self, input_ids: List[int], mm_inputs: MultimodalInputs):
def pad_input_ids(self, input_ids: array, mm_inputs: MultimodalInputs) -> array:
pattern = MultiModalityDataPaddingPatternMultimodalTokens()
return pattern.pad_input_tokens(input_ids, mm_inputs)
+2 -1
View File
@@ -18,6 +18,7 @@
"""Inference-only GLM-ASR-HF model compatible with HuggingFace weights."""
import logging
from array import array
from typing import Any, Iterable, List, Optional, Tuple
import torch
@@ -88,7 +89,7 @@ class GlmAsrForConditionalGeneration(nn.Module):
)
self.pattern = MultiModalityDataPaddingPatternMultimodalTokens()
def pad_input_ids(self, input_ids: List[int], mm_inputs: MultimodalInputs):
def pad_input_ids(self, input_ids: array, mm_inputs: MultimodalInputs) -> array:
return self.pattern.pad_input_tokens(input_ids, mm_inputs)
def get_audio_feature(self, items: List[MultimodalDataItem]) -> torch.Tensor:
+2 -1
View File
@@ -3,6 +3,7 @@ from __future__ import annotations
import copy
import logging
import re
from array import array
from typing import Iterable, Optional, Set, Tuple
import torch
@@ -1122,7 +1123,7 @@ class InklingForConditionalGeneration(nn.Module):
return 2
return 1
def pad_input_ids(self, input_ids: list[int], mm_inputs: MultimodalInputs):
def pad_input_ids(self, input_ids: array, mm_inputs: MultimodalInputs) -> array:
# The processor expands one placeholder per media item into a run of the same
# token id; the scheduler calls this to replace each run with the item's
# pad_value (radix hash), which _embed_mm then masks on to scatter the embeds.
+2 -1
View File
@@ -1,3 +1,4 @@
from array import array
from typing import Iterable, List, Optional, Tuple
import torch
@@ -154,7 +155,7 @@ class InternS1ForConditionalGeneration(nn.Module):
return hs
def pad_input_ids(self, input_ids: List[int], mm_inputs: MultimodalInputs):
def pad_input_ids(self, input_ids: array, mm_inputs: MultimodalInputs) -> array:
# Get all special token IDs
im_start_id: int = mm_inputs.im_start_id
im_end_id: int = mm_inputs.im_end_id
+2 -1
View File
@@ -1,5 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from array import array
from typing import Iterable, List, Optional, Tuple, Union
import torch
@@ -674,7 +675,7 @@ class InternVLChatModel(nn.Module):
return hidden_states
def pad_input_ids(self, input_ids: List[int], mm_inputs: MultimodalInputs):
def pad_input_ids(self, input_ids: array, mm_inputs: MultimodalInputs) -> array:
# Get all special token IDs
im_start_id: int = mm_inputs.im_start_id
im_end_id: int = mm_inputs.im_end_id
+2 -3
View File
@@ -1,4 +1,5 @@
import math
from array import array
from collections.abc import Iterable
import einops
@@ -133,9 +134,7 @@ class JetVLMForConditionalGeneration(nn.Module):
)
weight_loader(param, loaded_weight)
def pad_input_ids(
self, input_ids: list[int], mm_inputs: MultimodalInputs
) -> list[int]:
def pad_input_ids(self, input_ids: array, mm_inputs: MultimodalInputs) -> array:
pattern = MultiModalityDataPaddingPatternMultimodalTokens()
return pattern.pad_input_tokens(input_ids, mm_inputs)
+2 -1
View File
@@ -1,4 +1,5 @@
import logging
from array import array
from copy import deepcopy
from typing import Iterable, List, Optional, Sequence, Tuple
@@ -775,7 +776,7 @@ class KimiK25ForConditionalGeneration(nn.Module):
image_embeds = self.vision_tower(pixel_values, grid_thws)
return mm_projection_auto(self.mm_projector, image_embeds)
def pad_input_ids(self, input_ids: List[int], mm_inputs: MultimodalInputs):
def pad_input_ids(self, input_ids: array, mm_inputs: MultimodalInputs) -> array:
pattern = MultiModalityDataPaddingPatternMultimodalTokens()
return pattern.pad_input_tokens(input_ids, mm_inputs)
+2 -1
View File
@@ -9,6 +9,7 @@
import logging
import os
import re
from array import array
from collections.abc import Iterable
from functools import cached_property
from types import SimpleNamespace
@@ -3815,7 +3816,7 @@ class KimiK3ForConditionalGeneration(nn.Module):
image_embeds = self.vision_tower(pixel_values, grid_thws_host.to(device))
return self.mm_projector(image_embeds)
def pad_input_ids(self, input_ids: List[int], mm_inputs: MultimodalInputs):
def pad_input_ids(self, input_ids: array, mm_inputs: MultimodalInputs) -> array:
pattern = MultiModalityDataPaddingPatternMultimodalTokens()
return pattern.pad_input_tokens(input_ids, mm_inputs)
+2 -1
View File
@@ -44,6 +44,7 @@
import copy
import logging
import math
from array import array
from dataclasses import dataclass
from typing import Iterable, List, Optional, Tuple
@@ -192,7 +193,7 @@ class KimiVLForConditionalGeneration(nn.Module):
res = self.multi_modal_projector(image_features)
return res
def pad_input_ids(self, input_ids: List[int], mm_inputs: MultimodalInputs):
def pad_input_ids(self, input_ids: array, mm_inputs: MultimodalInputs) -> array:
pattern = MultiModalityDataPaddingPatternMultimodalTokens()
return pattern.pad_input_tokens(input_ids, mm_inputs)
+2 -3
View File
@@ -20,6 +20,7 @@ LFM2-VL is a vision-language model that combines:
"""
import logging
from array import array
from typing import Iterable, List, Optional, Tuple
import numpy as np
@@ -170,9 +171,7 @@ class Lfm2VlForConditionalGeneration(nn.Module):
self.logits_processor = LogitsProcessor(config.text_config)
def pad_input_ids(
self, input_ids: List[int], mm_inputs: MultimodalInputs
) -> List[int]:
def pad_input_ids(self, input_ids: array, mm_inputs: MultimodalInputs) -> array:
pattern = MultiModalityDataPaddingPatternMultimodalTokens()
result = pattern.pad_input_tokens(input_ids, mm_inputs)
return result
+2 -1
View File
@@ -31,6 +31,7 @@ References:
- https://huggingface.co/lightonai/LightOnOCR-2-1B
"""
from array import array
from dataclasses import fields
from typing import Iterable, List, Tuple
@@ -125,7 +126,7 @@ class LightOnOCRForConditionalGeneration(nn.Module):
quant_config=quant_config,
)
def pad_input_ids(self, input_ids: List[int], mm_inputs: MultimodalInputs):
def pad_input_ids(self, input_ids: array, mm_inputs: MultimodalInputs) -> array:
pattern = MultiModalityDataPaddingPatternMultimodalTokens()
return pattern.pad_input_tokens(input_ids, mm_inputs)
+1 -1
View File
@@ -650,7 +650,7 @@ class LlavaForConditionalGeneration(LlavaBaseForCausalLM):
def dtype(self):
return self.torch_dtype
def pad_input_ids(self, input_ids: List[int], image_inputs: MultimodalInputs):
def pad_input_ids(self, input_ids: array, image_inputs: MultimodalInputs) -> array:
if hasattr(self.vision_tower, "pad_input_ids"):
return self.vision_tower.pad_input_ids(input_ids, image_inputs)
else:
+2 -1
View File
@@ -19,6 +19,7 @@ pattern.
"""
import logging
from array import array
from typing import Any, Dict, Iterable, List, Optional, Set, Tuple
import torch
@@ -132,7 +133,7 @@ class LocateAnythingForConditionalGeneration(nn.Module):
assert isinstance(image_features, list)
return self.multi_modal_projector(torch.cat(image_features))
def pad_input_ids(self, input_ids: List[int], mm_inputs: MultimodalInputs):
def pad_input_ids(self, input_ids: array, mm_inputs: MultimodalInputs) -> array:
pattern = MultiModalityDataPaddingPatternMultimodalTokens()
return pattern.pad_input_tokens(input_ids, mm_inputs)
+2 -1
View File
@@ -1,6 +1,7 @@
import collections
import collections.abc
import logging
from array import array
from collections.abc import Callable, Sequence
from typing import Iterable, List, Optional, Tuple, TypeAlias, cast
@@ -502,7 +503,7 @@ class MiDashengLMModel(nn.Module):
self.logits_processor = self.language_model.logits_processor
self.quant_config = quant_config
def pad_input_ids(self, input_ids: List[int], mm_inputs: MultimodalInputs):
def pad_input_ids(self, input_ids: array, mm_inputs: MultimodalInputs) -> array:
"""Pad input IDs with multimodal tokens."""
pattern = MultiModalityDataPaddingPatternMultimodalTokens()
return pattern.pad_input_tokens(input_ids, mm_inputs)
+2 -1
View File
@@ -15,6 +15,7 @@
import logging
import math
import re
from array import array
from typing import Any, Dict, Iterable, List, Optional, Tuple, Union
import torch
@@ -1273,7 +1274,7 @@ class MiMoV2ForCausalLM(nn.Module, AudioEncoderMixin):
)
return self.model.get_input_embedding(input_ids)
def pad_input_ids(self, input_ids: List[int], mm_inputs: MultimodalInputs):
def pad_input_ids(self, input_ids: array, mm_inputs: MultimodalInputs) -> array:
pattern = MultiModalityDataPaddingPatternMultimodalTokens()
return pattern.pad_input_tokens(input_ids, mm_inputs)
+2 -1
View File
@@ -7,6 +7,7 @@ state_dict aligns 1:1 with ``self.named_parameters()``.
"""
import logging
from array import array
from typing import Any, Iterable, List, Optional, Tuple
import torch
@@ -56,7 +57,7 @@ class MiMoV2ASRForCausalLM(MiMoForCausalLM, AudioEncoderMixin):
super().__init__(config, quant_config=quant_config, prefix=prefix)
self.build_audio_encoder(MiMoAudioEncoderConfig(**config.audio_config))
def pad_input_ids(self, input_ids: List[int], mm_inputs: MultimodalInputs):
def pad_input_ids(self, input_ids: array, mm_inputs: MultimodalInputs) -> array:
pattern = MultiModalityDataPaddingPatternMultimodalTokens()
return pattern.pad_input_tokens(input_ids, mm_inputs)
+2 -1
View File
@@ -16,6 +16,7 @@
"""Inference-only MiniCPM-o model compatible with HuggingFace weights."""
import math
from array import array
from dataclasses import dataclass
from typing import Any, Iterable, List, Literal, Optional, Tuple, Union
@@ -1518,7 +1519,7 @@ class MiniCPMO(MiniCPMBaseModel):
return resampler.to(device=get_device(), dtype=torch.get_default_dtype())
def pad_input_ids(self, input_ids: List[int], mm_input: MultimodalInputs):
def pad_input_ids(self, input_ids: array, mm_input: MultimodalInputs) -> array:
# Get all special token IDs
im_start_id: int = mm_input.im_start_id
im_end_id: int = mm_input.im_end_id
+5 -4
View File
@@ -22,6 +22,7 @@
"""Inference-only MiniCPM-V model compatible with HuggingFace weights."""
import types
from array import array
from functools import partial
from itertools import chain
from typing import (
@@ -983,7 +984,7 @@ class MiniCPMV2_6(MiniCPMBaseModel):
)
return self.resampler(vision_embedding, tgt_sizes)
def pad_input_ids(self, input_ids: List[int], image_inputs: MultimodalInputs):
def pad_input_ids(self, input_ids: array, image_inputs: MultimodalInputs) -> array:
# Get all special token IDs
im_start_id: int = image_inputs.im_start_id
im_end_id: int = image_inputs.im_end_id
@@ -1149,7 +1150,7 @@ class MiniCPMV4_0(MiniCPMBaseModel):
)
return self.resampler(vision_embedding, tgt_sizes)
def pad_input_ids(self, input_ids: List[int], image_inputs: MultimodalInputs):
def pad_input_ids(self, input_ids: array, image_inputs: MultimodalInputs) -> array:
# Get all special token IDs
im_start_id: int = image_inputs.im_start_id
im_end_id: int = image_inputs.im_end_id
@@ -1319,7 +1320,7 @@ class MiniCPMV4_5(MiniCPMBaseModel):
)
return self.resampler(vision_embedding, tgt_sizes)
def pad_input_ids(self, input_ids: List[int], image_inputs: MultimodalInputs):
def pad_input_ids(self, input_ids: array, image_inputs: MultimodalInputs) -> array:
# Get all special token IDs
im_start_id: int = image_inputs.im_start_id
im_end_id: int = image_inputs.im_end_id
@@ -1544,7 +1545,7 @@ class MiniCPMV4_6(MiniCPMBaseModel):
# source. sglang's dispatcher routes by ``get_{modality}_feature``.
get_video_feature = get_image_feature
def pad_input_ids(self, input_ids: List[int], image_inputs: MultimodalInputs):
def pad_input_ids(self, input_ids: array, image_inputs: MultimodalInputs) -> array:
im_start_id: int = image_inputs.im_start_id
im_end_id: int = image_inputs.im_end_id
slice_start_id: int = image_inputs.slice_start_id
+2 -1
View File
@@ -1,6 +1,7 @@
# SPDX-License-Identifier: Apache-2.0
import logging
from array import array
from typing import Iterable, List, Optional, Tuple
import torch
@@ -195,7 +196,7 @@ class MiniMaxM3SparseForConditionalGeneration(nn.Module):
text_config
)
def pad_input_ids(self, input_ids: List[int], mm_inputs: MultimodalInputs):
def pad_input_ids(self, input_ids: array, mm_inputs: MultimodalInputs) -> array:
return MultiModalityDataPaddingPatternMultimodalTokens().pad_input_tokens(
input_ids, mm_inputs
)
+2 -1
View File
@@ -3,6 +3,7 @@ import logging
import math
import os
import re
from array import array
from collections.abc import Iterable
from typing import List, Optional, Set, Tuple
@@ -559,7 +560,7 @@ class Llama4ForConditionalGeneration(nn.Module):
except (OSError, json_lib.JSONDecodeError, KeyError):
return False
def pad_input_ids(self, input_ids: List[int], mm_inputs: MultimodalInputs):
def pad_input_ids(self, input_ids: array, mm_inputs: MultimodalInputs) -> array:
return self.padding_pattern.pad_input_tokens(input_ids, mm_inputs)
def get_image_feature(
+2 -1
View File
@@ -13,6 +13,7 @@
# ==============================================================================
import logging
from array import array
from typing import Iterable, List, Optional, Tuple
import torch
@@ -977,7 +978,7 @@ class MuseGlimmerForConditionalGeneration(MuseGlimmerForCausalLM):
config.hidden_size, eps=config.rms_norm_eps, has_weight=False
)
def pad_input_ids(self, input_ids: List[int], mm_inputs: MultimodalInputs):
def pad_input_ids(self, input_ids: array, mm_inputs: MultimodalInputs) -> array:
return MultiModalityDataPaddingPatternMultimodalTokens().pad_input_tokens(
input_ids, mm_inputs
)
+2 -1
View File
@@ -16,6 +16,7 @@
# Adapted from https://github.com/vllm-project/vllm/blob/main/vllm/model_executor/models/nano_nemotron_vl.py
import logging
from array import array
from collections import deque
from typing import Iterable
@@ -116,7 +117,7 @@ class NemotronH_Nano_VL_V2(EVS):
self.config = config
def pad_input_ids(self, input_ids: list[int], mm_inputs: MultimodalInputs):
def pad_input_ids(self, input_ids: array, mm_inputs: MultimodalInputs) -> array:
im_start_id: int = mm_inputs.im_start_id
im_end_id: int = mm_inputs.im_end_id
+2 -3
View File
@@ -1,5 +1,6 @@
import itertools
import math
from array import array
from collections.abc import Iterable
from typing import Any
@@ -218,9 +219,7 @@ class NVILAForConditionalGeneration(nn.Module):
)
weight_loader(param, loaded_weight)
def pad_input_ids(
self, input_ids: list[int], mm_inputs: MultimodalInputs
) -> list[int]:
def pad_input_ids(self, input_ids: array, mm_inputs: MultimodalInputs) -> array:
pattern = MultiModalityDataPaddingPatternMultimodalTokens()
return pattern.pad_input_tokens(input_ids, mm_inputs)
+2 -3
View File
@@ -1,4 +1,5 @@
import math
from array import array
from collections.abc import Iterable
from typing import Any
@@ -178,9 +179,7 @@ class NVILALiteForConditionalGeneration(nn.Module):
)
weight_loader(param, loaded_weight)
def pad_input_ids(
self, input_ids: list[int], mm_inputs: MultimodalInputs
) -> list[int]:
def pad_input_ids(self, input_ids: array, mm_inputs: MultimodalInputs) -> array:
pattern = MultiModalityDataPaddingPatternMultimodalTokens()
return pattern.pad_input_tokens(input_ids, mm_inputs)
+2 -1
View File
@@ -24,6 +24,7 @@ projections run once for the batch instead of once per image.
"""
import itertools
from array import array
from collections.abc import Iterable
from typing import List, Optional, Set, Tuple
@@ -584,7 +585,7 @@ class PaddleOCRVLForConditionalGeneration(Ernie4_5_ForCausalLM):
)
self.is_mrope_enabled = "mrope_section" in (self.config.rope_scaling or {})
def pad_input_ids(self, input_ids: List[int], mm_inputs: MultimodalInputs):
def pad_input_ids(self, input_ids: array, mm_inputs: MultimodalInputs) -> array:
pattern = MultiModalityDataPaddingPatternMultimodalTokens()
return pattern.pad_input_tokens(input_ids, mm_inputs)
+2 -1
View File
@@ -20,6 +20,7 @@
import logging
import math
import re
from array import array
from collections.abc import Iterable
from typing import List, Optional, Tuple
@@ -475,7 +476,7 @@ class Phi4MMForCausalLM(nn.Module):
return hidden_states
def pad_input_ids(self, input_ids: List[int], mm_inputs: MultimodalInputs):
def pad_input_ids(self, input_ids: array, mm_inputs: MultimodalInputs) -> array:
pattern = MultiModalityDataPaddingPatternMultimodalTokens()
return pattern.pad_input_tokens(input_ids, mm_inputs)
+3 -2
View File
@@ -16,6 +16,7 @@
Using mistral-community/pixtral-12b as reference.
"""
from array import array
from dataclasses import dataclass, fields
from typing import Iterable, List, Optional, Set, Tuple, Union
@@ -137,7 +138,7 @@ class PixtralForConditionalGeneration(nn.Module):
self.vision_args, dim=self.config.text_config.hidden_size
)
def pad_input_ids(self, input_ids: List[int], mm_inputs: MultimodalInputs):
def pad_input_ids(self, input_ids: array, mm_inputs: MultimodalInputs) -> array:
pattern = MultiModalityDataPaddingPatternMultimodalTokens()
return pattern.pad_input_tokens(input_ids, mm_inputs)
@@ -858,7 +859,7 @@ class PixtralHFVisionModel(nn.Module):
DEFAULT_IMAGE_TOKEN_ID = 10
def pad_input_ids(self, input_ids: List[int], mm_inputs: MultimodalInputs):
def pad_input_ids(self, input_ids: array, mm_inputs: MultimodalInputs) -> array:
return self.input_padder.pad_input_tokens(input_ids, mm_inputs)
def __init__(
+2 -1
View File
@@ -1,4 +1,5 @@
import copy
from array import array
from typing import Iterable, List, Optional, Set, Tuple
import torch
@@ -101,7 +102,7 @@ class POINTSV15ChatModel(nn.Module):
prefix=add_prefix("vision_projector", prefix),
)
def pad_input_ids(self, input_ids: List[int], mm_inputs: MultimodalInputs):
def pad_input_ids(self, input_ids: array, mm_inputs: MultimodalInputs) -> array:
pattern = MultiModalityDataPaddingPatternMultimodalTokens()
return pattern.pad_input_tokens(input_ids, mm_inputs)
+2 -1
View File
@@ -25,6 +25,7 @@
import logging
import re
from array import array
from functools import partial
from typing import Iterable, List, Optional, Tuple, Type
@@ -767,7 +768,7 @@ class Qwen2_5_VLForConditionalGeneration(nn.Module):
# For EAGLE3 support
self.capture_aux_hidden_states = False
def pad_input_ids(self, input_ids: List[int], mm_inputs: MultimodalInputs):
def pad_input_ids(self, input_ids: array, mm_inputs: MultimodalInputs) -> array:
pattern = MultiModalityDataPaddingPatternMultimodalTokens()
return pattern.pad_input_tokens(input_ids, mm_inputs)
+2 -1
View File
@@ -24,6 +24,7 @@
"""Inference-only Qwen2-Audio model compatible with HuggingFace weights."""
import logging
from array import array
from typing import Any, Iterable, List, Optional, Tuple
import torch
@@ -100,7 +101,7 @@ class Qwen2AudioForConditionalGeneration(nn.Module):
)
self.pattern = MultiModalityDataPaddingPatternMultimodalTokens()
def pad_input_ids(self, input_ids: List[int], mm_inputs: MultimodalInputs):
def pad_input_ids(self, input_ids: array, mm_inputs: MultimodalInputs) -> array:
return self.pattern.pad_input_tokens(input_ids, mm_inputs)
def get_audio_feature(self, items: List[MultimodalDataItem]) -> torch.Tensor:
+2 -1
View File
@@ -24,6 +24,7 @@
"""Inference-only Qwen2-VL model compatible with HuggingFace weights."""
import logging
from array import array
from functools import lru_cache, partial
from typing import Iterable, List, Optional, Tuple, Type, TypedDict
@@ -492,7 +493,7 @@ class Qwen2VLForConditionalGeneration(nn.Module):
self.logits_processor = LogitsProcessor(config)
self.pooler = Pooler(pooling_type=PoolingType.LAST, normalize=True)
def pad_input_ids(self, input_ids: List[int], mm_inputs: MultimodalInputs):
def pad_input_ids(self, input_ids: array, mm_inputs: MultimodalInputs) -> array:
pattern = MultiModalityDataPaddingPatternMultimodalTokens()
return pattern.pad_input_tokens(input_ids, mm_inputs)
+2 -1
View File
@@ -1,6 +1,7 @@
"""Qwen3-ASR model compatible with HuggingFace weights"""
import logging
from array import array
from typing import Any, Iterable, List, Optional, Tuple
import torch
@@ -66,7 +67,7 @@ class Qwen3ASRForConditionalGeneration(nn.Module):
)
self.pattern = MultiModalityDataPaddingPatternMultimodalTokens()
def pad_input_ids(self, input_ids: List[int], mm_inputs: MultimodalInputs):
def pad_input_ids(self, input_ids: array, mm_inputs: MultimodalInputs) -> array:
return self.pattern.pad_input_tokens(input_ids, mm_inputs)
def get_audio_feature(self, items: List[MultimodalDataItem]) -> torch.Tensor:
+2 -1
View File
@@ -16,6 +16,7 @@
import logging
import re
from array import array
from collections import defaultdict
from functools import lru_cache, partial
from typing import Callable, Iterable, List, Optional, Tuple, Union
@@ -1420,7 +1421,7 @@ class Qwen3VLForConditionalGeneration(nn.Module):
cfg = getattr(model, "config", None)
return int(getattr(cfg, "num_hidden_layers", 0))
def pad_input_ids(self, input_ids: List[int], mm_inputs: MultimodalInputs):
def pad_input_ids(self, input_ids: array, mm_inputs: MultimodalInputs) -> array:
if mm_inputs and mm_inputs.mm_items:
_require_vision(self)
pattern = MultiModalityDataPaddingPatternMultimodalTokens()
@@ -14,6 +14,7 @@
"""Inference-only Sarashina2Vision model compatible with HuggingFace weights."""
import logging
from array import array
from typing import Iterable, List, Optional, Tuple
import torch
@@ -107,7 +108,7 @@ class Sarashina2VisionForCausalLM(nn.Module):
self.logits_processor = LogitsProcessor(config)
self.pooler = Pooler(pooling_type=PoolingType.LAST, normalize=True)
def pad_input_ids(self, input_ids: List[int], mm_inputs: MultimodalInputs):
def pad_input_ids(self, input_ids: array, mm_inputs: MultimodalInputs) -> array:
"""Pad input tokens with multimodal data hashes for RadixAttention."""
pattern = MultiModalityDataPaddingPatternMultimodalTokens()
return pattern.pad_input_tokens(input_ids, mm_inputs)
+2 -1
View File
@@ -1,5 +1,6 @@
import logging
import math
from array import array
from math import sqrt
from typing import Any, Dict, Iterable, List, Optional, Tuple
@@ -895,7 +896,7 @@ class Step3VLForConditionalGeneration(nn.Module):
)
return self._flatten_embeddings(merged_image_features)
def pad_input_ids(self, input_ids: List[int], mm_inputs: MultimodalInputs):
def pad_input_ids(self, input_ids: array, mm_inputs: MultimodalInputs) -> array:
pattern = MultiModalityDataPaddingPatternMultimodalTokens()
return pattern.pad_input_tokens(input_ids, mm_inputs)
+2 -1
View File
@@ -2,6 +2,7 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""This is basically a copy from perception_models/core/vision_encoder/pe.py"""
from array import array
from functools import partial
from typing import Callable, Iterable, List, Optional, Tuple
@@ -570,7 +571,7 @@ class StepVLForConditionalGeneration(nn.Module):
)
return self._flatten_embeddings(merged_image_features)
def pad_input_ids(self, input_ids: List[int], mm_inputs: MultimodalInputs):
def pad_input_ids(self, input_ids: array, mm_inputs: MultimodalInputs) -> array:
pattern = MultiModalityDataPaddingPatternMultimodalTokens()
return pattern.pad_input_tokens(input_ids, mm_inputs)
+2 -1
View File
@@ -1,3 +1,4 @@
from array import array
from typing import Iterable, List, Optional, Tuple
import torch
@@ -128,7 +129,7 @@ class Step3p7ForConditionalGeneration(nn.Module):
)
return self._flatten_embeddings(merged_image_features)
def pad_input_ids(self, input_ids: List[int], mm_inputs: MultimodalInputs):
def pad_input_ids(self, input_ids: array, mm_inputs: MultimodalInputs) -> array:
pattern = MultiModalityDataPaddingPatternMultimodalTokens()
return pattern.pad_input_tokens(input_ids, mm_inputs)
+2 -1
View File
@@ -21,6 +21,7 @@
import inspect
import logging
import re
from array import array
from collections.abc import Iterable, Mapping
from contextlib import contextmanager
from typing import List, Literal, Optional, Tuple, Union
@@ -1368,7 +1369,7 @@ class MultiModalMixin:
rope_type = str(getattr(self.text_config, "rope_type", "")).lower()
return "mrope" in rope_type
def pad_input_ids(self, input_ids: list[int], mm_inputs: MultimodalInputs):
def pad_input_ids(self, input_ids: array, mm_inputs: MultimodalInputs) -> array:
return input_ids
def _get_modality_encoder(self, modality_name: str):
+2 -1
View File
@@ -1,6 +1,7 @@
"""Standalone UNLIMITED-OCR model (SAM + CLIP vision encoders, Deepseek backbone)."""
import logging
from array import array
from typing import Iterable, List, Optional, Set, Tuple, TypeAlias, Union
import torch
@@ -311,7 +312,7 @@ class UnlimitedOCRForCausalLM(nn.Module):
)
return inputs_embeds
def pad_input_ids(self, input_ids: List[int], mm_inputs: MultimodalInputs):
def pad_input_ids(self, input_ids: array, mm_inputs: MultimodalInputs) -> array:
"""Pad input token IDs with multimodal placeholder tokens."""
pattern = MultiModalityDataPaddingPatternMultimodalTokens()
return pattern.pad_input_tokens(input_ids, mm_inputs)
+2 -1
View File
@@ -9,6 +9,7 @@
"""Inference-only Voxtral (speech-to-text) model."""
import math
from array import array
from typing import Any, Iterable, List, Optional, Tuple
import torch
@@ -337,7 +338,7 @@ class VoxtralForConditionalGeneration(nn.Module):
return results
def pad_input_ids(self, input_ids: List[int], mm_inputs: MultimodalInputs):
def pad_input_ids(self, input_ids: array, mm_inputs: MultimodalInputs) -> array:
return self.pattern.pad_input_tokens(input_ids, mm_inputs)
def get_audio_feature(self, items: List[MultimodalDataItem]) -> torch.Tensor:
+6 -4
View File
@@ -15,6 +15,8 @@
# ==============================================================================
# Adapted from https://github.com/vllm-project/vllm/blob/main/vllm/multimodal/evs.py
from array import array
import torch
@@ -122,8 +124,7 @@ def tokens_per_frame(
def replace_offsets_with_tokens_per_frame(
*,
pre_chunked_input_ids: list[int],
pre_chunked_input_ids: list[int] | array,
num_tokens_per_frame: list[int],
frame_offsets_inclusive: list[tuple[int, int]],
filler_token_id: int,
@@ -157,12 +158,13 @@ def replace_offsets_with_tokens_per_frame(
... filler_token_id=0,
... ) == [0, 1, 4, 0, 0, 0, 0, 5, 9, 10]
"""
assert isinstance(pre_chunked_input_ids, list)
# Encoder metadata uses lists; model padding can replace it with packed tokens.
assert isinstance(pre_chunked_input_ids, (list, array))
ids = pre_chunked_input_ids
if len(frame_offsets_inclusive) == 1:
"""There might be no frame separators, in which case there will be one contiguous span of tokens"""
final = ids[0 : frame_offsets_inclusive[0][0]]
final = list(ids[0 : frame_offsets_inclusive[0][0]])
frames = [filler_token_id] * sum(num_tokens_per_frame)
final.extend(frames)
else:
+7
View File
@@ -1281,6 +1281,13 @@ class Range(NamedTuple):
return self.end - self.start
def assert_int64_array(values: array, name: str) -> None:
"""Require a signed int64 array suitable for zero-copy tensor views."""
assert (
isinstance(values, array) and values.typecode == "q" and values.itemsize == 8
), f"{name} must be array('q') with 8-byte items"
def flatten_arrays_to_pinned_cpu(parts: List[array[int]], pin: bool) -> torch.Tensor:
"""Flatten array.array('q') buffers into one int64 CPU tensor.