[MM] Keep scheduler padding in packed token arrays (#40357)
This commit is contained in:
@@ -0,0 +1,198 @@
|
||||
"""CPU comparison of baseline and optimized offset-based scheduler padding.
|
||||
|
||||
Run from the repository root:
|
||||
PYTHONPATH=python python benchmark/io/bench_mm_token_padding.py --baseline-ref BASE_SHA
|
||||
|
||||
Inputs are synthetic. Both paths include the scheduler's final ownership copy.
|
||||
This measures padding CPU time, not model forward time or end-to-end TTFT.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import ast
|
||||
import gc
|
||||
import json
|
||||
import platform
|
||||
import statistics
|
||||
import subprocess
|
||||
import time
|
||||
from array import array
|
||||
from pathlib import Path
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.srt.managers import mm_utils
|
||||
from sglang.srt.managers.mm_utils import (
|
||||
MultiModalityDataPaddingPatternMultimodalTokens,
|
||||
duplicate_pad_mm_input_ids,
|
||||
)
|
||||
from sglang.srt.managers.schedule_batch import (
|
||||
Modality,
|
||||
MultimodalDataItem,
|
||||
MultimodalInputs,
|
||||
)
|
||||
|
||||
|
||||
def load_baseline(ref):
|
||||
"""Load the exact baseline helper classes without replacing installed SGLang."""
|
||||
root = Path(__file__).resolve().parents[2]
|
||||
commit = subprocess.check_output(
|
||||
["git", "rev-parse", "--verify", f"{ref}^{{commit}}"], cwd=root, text=True
|
||||
).strip()
|
||||
path = "python/sglang/srt/managers/mm_utils.py"
|
||||
source = subprocess.check_output(
|
||||
["git", "show", f"{commit}:{path}"], cwd=root, text=True
|
||||
)
|
||||
names = {
|
||||
"MultiModalityDataPaddingPattern",
|
||||
"MultiModalityDataPaddingPatternMultimodalTokens",
|
||||
}
|
||||
classes = [
|
||||
node
|
||||
for node in ast.parse(source).body
|
||||
if isinstance(node, ast.ClassDef) and node.name in names
|
||||
]
|
||||
if {node.name for node in classes} != names:
|
||||
raise ValueError("Baseline does not contain the expected padding helpers")
|
||||
namespace = dict(vars(mm_utils))
|
||||
# Reuse dependency imports, but compile the original method bodies verbatim.
|
||||
exec(
|
||||
compile(ast.Module(body=classes, type_ignores=[]), f"{commit}:{path}", "exec"),
|
||||
namespace,
|
||||
)
|
||||
return commit, namespace["MultiModalityDataPaddingPatternMultimodalTokens"]()
|
||||
|
||||
|
||||
def make_case(num_tokens, num_items, span_tokens):
|
||||
if num_items and num_tokens < 3 * num_items:
|
||||
raise ValueError(
|
||||
"Each item needs three tokens for its span and surrounding text"
|
||||
)
|
||||
input_ids = array("q", range(257, 257 + num_tokens))
|
||||
items = []
|
||||
stride = num_tokens // max(num_items, 1)
|
||||
for index in range(num_items):
|
||||
start = index * stride + 1
|
||||
end = start + min(span_tokens, stride - 2) - 1
|
||||
items.append(
|
||||
MultimodalDataItem(
|
||||
modality=Modality.IMAGE,
|
||||
pad_value=2**40 + index,
|
||||
offsets=[(start, end)],
|
||||
)
|
||||
)
|
||||
return input_ids, MultimodalInputs(mm_items=items, im_token_id=1)
|
||||
|
||||
|
||||
def benchmark_case(num_tokens, num_items, args, baseline):
|
||||
pattern = MultiModalityDataPaddingPatternMultimodalTokens()
|
||||
input_ids, mm_inputs = make_case(num_tokens, num_items, args.span_tokens)
|
||||
original = input_ids.tobytes()
|
||||
|
||||
def baseline_padding():
|
||||
return array("q", baseline.pad_input_tokens(input_ids, mm_inputs))
|
||||
|
||||
def optimized_padding():
|
||||
return duplicate_pad_mm_input_ids(
|
||||
input_ids, mm_inputs, pattern.pad_input_tokens
|
||||
)
|
||||
|
||||
expected = baseline_padding()
|
||||
actual = optimized_padding()
|
||||
assert actual == expected, "Token values differ"
|
||||
assert input_ids.tobytes() == original, "Original input was mutated"
|
||||
assert actual is not input_ids, "Scheduler result aliases the original input"
|
||||
|
||||
functions = {"baseline": baseline_padding, "optimized": optimized_padding}
|
||||
timings = {name: [] for name in functions}
|
||||
for _ in range(args.warmups):
|
||||
for function in functions.values():
|
||||
function()
|
||||
gc_was_enabled = gc.isenabled()
|
||||
gc.disable()
|
||||
try:
|
||||
for trial in range(args.trials):
|
||||
order = (
|
||||
("baseline", "optimized") if trial % 2 else ("optimized", "baseline")
|
||||
)
|
||||
for name in order:
|
||||
start = time.perf_counter_ns()
|
||||
result = functions[name]()
|
||||
elapsed = time.perf_counter_ns() - start
|
||||
timings[name].append(elapsed / 1e6)
|
||||
del result
|
||||
finally:
|
||||
if gc_was_enabled:
|
||||
gc.enable()
|
||||
assert input_ids.tobytes() == original
|
||||
medians = {name: statistics.median(samples) for name, samples in timings.items()}
|
||||
return {
|
||||
"tokens": num_tokens,
|
||||
"items": num_items,
|
||||
"baseline_ms": medians["baseline"],
|
||||
"optimized_ms": medians["optimized"],
|
||||
"speedup": medians["baseline"] / medians["optimized"],
|
||||
}
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument(
|
||||
"--baseline-ref", required=True, help="Local pre-change git ref"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--tokens", nargs="+", type=int, default=[128, 4096, 16384, 131072]
|
||||
)
|
||||
parser.add_argument("--items", nargs="+", type=int, default=[0, 1, 10])
|
||||
parser.add_argument("--span-tokens", type=int, default=32)
|
||||
parser.add_argument("--trials", type=int, default=41)
|
||||
parser.add_argument("--warmups", type=int, default=5)
|
||||
parser.add_argument("--threads", type=int, default=1)
|
||||
parser.add_argument("--json", action="store_true")
|
||||
args = parser.parse_args()
|
||||
if (
|
||||
min(args.tokens + args.items) < 0
|
||||
or min(args.span_tokens, args.trials, args.threads) < 1
|
||||
or args.warmups < 0
|
||||
):
|
||||
parser.error("Lengths/items/warmups must be nonnegative; other counts positive")
|
||||
if any(n < 3 * items for n in args.tokens for items in args.items):
|
||||
parser.error("Each token length must accommodate at least 3 tokens per item")
|
||||
torch.set_num_threads(args.threads)
|
||||
baseline_commit, baseline = load_baseline(args.baseline_ref)
|
||||
rows = [
|
||||
benchmark_case(tokens, items, args, baseline)
|
||||
for tokens in args.tokens
|
||||
for items in args.items
|
||||
]
|
||||
if args.json:
|
||||
print(
|
||||
json.dumps(
|
||||
{
|
||||
"baseline_commit": baseline_commit,
|
||||
"python": platform.python_version(),
|
||||
"torch": torch.__version__,
|
||||
"architecture": platform.machine(),
|
||||
"threads": args.threads,
|
||||
"trials": args.trials,
|
||||
"warmups": args.warmups,
|
||||
"span_tokens": args.span_tokens,
|
||||
"results": rows,
|
||||
},
|
||||
indent=2,
|
||||
)
|
||||
)
|
||||
return
|
||||
print("Offset-based CPU padding; median milliseconds; synthetic inputs")
|
||||
print(
|
||||
f"{'Tokens':>8} {'Items':>6} {'Baseline (ms)':>14} {'Optimized (ms)':>14} {'Speedup':>9}"
|
||||
)
|
||||
for row in rows:
|
||||
print(
|
||||
f"{row['tokens']:>8} {row['items']:>6} "
|
||||
f"{row['baseline_ms']:>14.4f} {row['optimized_ms']:>14.4f} "
|
||||
f"{row['speedup']:>8.2f}x"
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -9,9 +9,10 @@ import os
|
||||
import pickle
|
||||
import sys
|
||||
from abc import abstractmethod
|
||||
from array import array
|
||||
from collections import defaultdict
|
||||
from multiprocessing import shared_memory
|
||||
from typing import Any, Dict, List, Optional, Tuple
|
||||
from typing import Any, Callable, Dict, List, Optional, Tuple
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
@@ -49,7 +50,11 @@ from sglang.srt.runtime_context import (
|
||||
get_server_args,
|
||||
get_serving,
|
||||
)
|
||||
from sglang.srt.utils import flatten_nested_list, print_warning_once
|
||||
from sglang.srt.utils import (
|
||||
assert_int64_array,
|
||||
flatten_nested_list,
|
||||
print_warning_once,
|
||||
)
|
||||
from sglang.srt.utils.stale_shm_cleanup import make_shm_name
|
||||
from sglang.utils import logger
|
||||
|
||||
@@ -232,6 +237,19 @@ class TransportProxyTensor(torch.Tensor):
|
||||
return self._metadata.get("transport_mode", "default")
|
||||
|
||||
|
||||
def duplicate_pad_mm_input_ids(
|
||||
input_ids: array,
|
||||
mm_inputs: MultimodalInputs,
|
||||
pad_input_ids_func: Callable[[array, MultimodalInputs], array],
|
||||
) -> array:
|
||||
"""Validate and duplicate model-padded tokens for scheduler ownership."""
|
||||
padded_ids = pad_input_ids_func(input_ids, mm_inputs)
|
||||
assert_int64_array(padded_ids, "pad_input_ids result")
|
||||
# Preserve scheduler ownership: no-op models return the unpadded input,
|
||||
# and other models can retain the result in multimodal metadata.
|
||||
return padded_ids[:]
|
||||
|
||||
|
||||
class MultiModalityDataPaddingPattern:
|
||||
"""
|
||||
Data tokens (like image tokens) often need special handling during padding
|
||||
@@ -240,11 +258,10 @@ class MultiModalityDataPaddingPattern:
|
||||
"""
|
||||
|
||||
@abstractmethod
|
||||
def pad_input_tokens(
|
||||
self, input_ids: List[int], mm_inputs: MultimodalInputs
|
||||
) -> List[int]:
|
||||
def pad_input_tokens(self, input_ids: array, mm_inputs: MultimodalInputs) -> array:
|
||||
"""
|
||||
Pad the input ids sequence containing data tokens, and replace them with pad_values
|
||||
Replace data tokens with pad_values using native int64 arrays.
|
||||
Implementations must not mutate the original input.
|
||||
"""
|
||||
pass
|
||||
|
||||
@@ -273,12 +290,11 @@ class MultiModalityDataPaddingPatternTokenPairs(MultiModalityDataPaddingPattern)
|
||||
s for s, _e in data_token_pairs
|
||||
]
|
||||
|
||||
def pad_input_tokens(
|
||||
self, input_ids: List[int], mm_inputs: MultimodalInputs
|
||||
) -> List[int]:
|
||||
def pad_input_tokens(self, input_ids: array, mm_inputs: MultimodalInputs) -> array:
|
||||
"""
|
||||
This function will replace the data-tokens in between with pad_values accordingly
|
||||
"""
|
||||
assert_int64_array(input_ids, "input_ids")
|
||||
pad_values = [item.pad_value for item in mm_inputs.mm_items]
|
||||
data_token_pairs = self.data_token_id_pairs
|
||||
mm_inputs.data_offsets = []
|
||||
@@ -292,7 +308,7 @@ class MultiModalityDataPaddingPatternTokenPairs(MultiModalityDataPaddingPattern)
|
||||
start_token_ids = {s for s, _e in data_token_pairs}
|
||||
end_tokens_ids = {e for _s, e in data_token_pairs}
|
||||
|
||||
padded_ids = []
|
||||
padded_ids = array("q")
|
||||
last_idx = 0
|
||||
data_idx = -1
|
||||
|
||||
@@ -314,7 +330,8 @@ class MultiModalityDataPaddingPatternTokenPairs(MultiModalityDataPaddingPattern)
|
||||
|
||||
num_tokens = end_idx - start_idx - 1
|
||||
pad_value = pad_values[data_idx]
|
||||
padded_ids.extend([pad_value] * num_tokens)
|
||||
if num_tokens > 0:
|
||||
padded_ids.extend(array("q", [pad_value]) * num_tokens)
|
||||
|
||||
last_idx = end_idx
|
||||
|
||||
@@ -329,17 +346,19 @@ class MultiModalityDataPaddingPatternMultimodalTokens(MultiModalityDataPaddingPa
|
||||
e.g. <image><image>....<image>, or <audio><audio>...<audio>
|
||||
"""
|
||||
|
||||
def pad_input_tokens(
|
||||
self, input_ids: List[int], mm_inputs: MultimodalInputs
|
||||
) -> List[int]:
|
||||
def pad_input_tokens(self, input_ids: array, mm_inputs: MultimodalInputs) -> array:
|
||||
"""
|
||||
Replaces multimodal tokens in input_ids with corresponding pad_values from mm_items.
|
||||
Each modality (image, audio, video) is handled separately based on its token_id.
|
||||
"""
|
||||
assert_int64_array(input_ids, "input_ids")
|
||||
if not input_ids or not mm_inputs.mm_items:
|
||||
return input_ids
|
||||
|
||||
input_ids_tensor = torch.as_tensor(input_ids)
|
||||
# as_tensor(array) walks and boxes every token. View a bulk copy instead,
|
||||
# keeping the original request's unpadded token buffer unchanged.
|
||||
padded_input_ids = input_ids[:]
|
||||
input_ids_tensor = torch.frombuffer(padded_input_ids, dtype=torch.int64)
|
||||
|
||||
# Replace multimodal tokens using per-item offsets
|
||||
items_by_modality = defaultdict(list)
|
||||
@@ -362,8 +381,7 @@ class MultiModalityDataPaddingPatternMultimodalTokens(MultiModalityDataPaddingPa
|
||||
for offset in items[i].offsets:
|
||||
input_ids_tensor[offset[0] : offset[1] + 1] = item.pad_value
|
||||
|
||||
ret_input_ids = input_ids_tensor.tolist()
|
||||
return ret_input_ids
|
||||
return padded_input_ids
|
||||
|
||||
|
||||
# masked_scatter_ materializes the expanded [num_tokens, hidden] bool mask plus
|
||||
|
||||
@@ -193,6 +193,7 @@ from sglang.srt.managers.min_free_slots_delayer import (
|
||||
MinFreeSlotsDelayer,
|
||||
resolve_min_free_slots,
|
||||
)
|
||||
from sglang.srt.managers.mm_utils import duplicate_pad_mm_input_ids
|
||||
from sglang.srt.managers.multimodal_processor import get_mm_processor, import_processors
|
||||
from sglang.srt.managers.overlap_utils import (
|
||||
RelayPayload,
|
||||
@@ -1214,7 +1215,6 @@ class Scheduler(
|
||||
)
|
||||
self.dp_tp_cpu_group = self.dp_tp_group.cpu_group
|
||||
|
||||
# TODO(Jialin): Migrate pad_input_ids implementations to return array.
|
||||
self.pad_input_ids_func = self.tp_worker.get_pad_input_ids_func()
|
||||
set_random_seed(self.random_seed)
|
||||
|
||||
@@ -2994,8 +2994,8 @@ class Scheduler(
|
||||
not self._try_apply_padded_mm_input_ids(recv_req, req, image_inputs)
|
||||
and self.pad_input_ids_func
|
||||
):
|
||||
req.origin_input_ids = array(
|
||||
"q", self.pad_input_ids_func(req.origin_input_ids, image_inputs)
|
||||
req.origin_input_ids = duplicate_pad_mm_input_ids(
|
||||
req.origin_input_ids, image_inputs, self.pad_input_ids_func
|
||||
)
|
||||
req.extend_image_inputs(image_inputs)
|
||||
self._maybe_compute_mrope_positions(req)
|
||||
@@ -3468,9 +3468,8 @@ class Scheduler(
|
||||
not self._try_apply_padded_mm_input_ids(recv_req, req, image_inputs)
|
||||
and self.pad_input_ids_func
|
||||
):
|
||||
# See companion call site above for the array.array wrap rationale.
|
||||
req.origin_input_ids = array(
|
||||
"q", self.pad_input_ids_func(req.origin_input_ids, image_inputs)
|
||||
req.origin_input_ids = duplicate_pad_mm_input_ids(
|
||||
req.origin_input_ids, image_inputs, self.pad_input_ids_func
|
||||
)
|
||||
|
||||
req.extend_image_inputs(image_inputs)
|
||||
|
||||
@@ -55,6 +55,7 @@ from sglang.srt.mem_cache.unified_cache.unified_tree_core_interface import (
|
||||
UnifiedTreeCoreInterface,
|
||||
)
|
||||
from sglang.srt.runtime_context import get_exec, mamba_cache_chunk_size
|
||||
from sglang.srt.utils import assert_int64_array
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from sglang.srt.managers.schedule_batch import Req
|
||||
@@ -72,9 +73,7 @@ def _radix_key_buffer(key: RadixKey) -> array:
|
||||
"""The key's token ids honoring `limit`; view-independent since the
|
||||
binding derives its own atoms."""
|
||||
token_ids = key.raw_token_ids()
|
||||
assert isinstance(token_ids, array) and token_ids.typecode == "q", (
|
||||
f"tree keys must carry array('q') token ids, got {type(token_ids).__name__}"
|
||||
)
|
||||
assert_int64_array(token_ids, "tree key token ids")
|
||||
return token_ids
|
||||
|
||||
|
||||
|
||||
@@ -14,6 +14,7 @@
|
||||
"""Legacy Bailing multimodal wrappers for image and video inference."""
|
||||
|
||||
import logging
|
||||
from array import array
|
||||
from typing import Iterable, List, Optional, Set, Tuple
|
||||
|
||||
import torch
|
||||
@@ -122,7 +123,7 @@ class BailingMMNativeForConditionalGeneration(nn.Module):
|
||||
def get_input_embeddings(self):
|
||||
return self.model.get_input_embeddings()
|
||||
|
||||
def pad_input_ids(self, input_ids: List[int], mm_inputs: MultimodalInputs):
|
||||
def pad_input_ids(self, input_ids: array, mm_inputs: MultimodalInputs) -> array:
|
||||
return self.pattern.pad_input_tokens(input_ids, mm_inputs)
|
||||
|
||||
def _get_vision_feature(
|
||||
|
||||
@@ -14,6 +14,7 @@
|
||||
"""SGLang implementation of Bailing/Ling 3 VL image and video inference."""
|
||||
|
||||
import logging
|
||||
from array import array
|
||||
from typing import Iterable, List, Optional, Set, Tuple
|
||||
|
||||
import torch
|
||||
@@ -128,7 +129,7 @@ class BailingMoeV3VLForConditionalGeneration(nn.Module):
|
||||
def get_input_embeddings(self):
|
||||
return self.model.get_input_embeddings()
|
||||
|
||||
def pad_input_ids(self, input_ids: List[int], mm_inputs: MultimodalInputs):
|
||||
def pad_input_ids(self, input_ids: array, mm_inputs: MultimodalInputs) -> array:
|
||||
return self.pattern.pad_input_tokens(input_ids, mm_inputs)
|
||||
|
||||
def _materialize_items(self, items: List[MultimodalDataItem]) -> torch.Tensor:
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
# Adapted from
|
||||
# https://github.com/huggingface/transformers/blob/af9b2eaa54c150741f298d6db939af6328e1dc38/src/transformers/models/clip/modeling_clip.py
|
||||
|
||||
from array import array
|
||||
from functools import partial
|
||||
from typing import Iterable, List, Optional, Tuple, Type, Union
|
||||
|
||||
@@ -580,7 +581,7 @@ class CLIPModel(nn.Module):
|
||||
embeddings=self.text_projection(pooled_output.embeddings)
|
||||
)
|
||||
|
||||
def pad_input_ids(self, input_ids: List[int], image_inputs: MultimodalInputs):
|
||||
def pad_input_ids(self, input_ids: array, image_inputs: MultimodalInputs) -> array:
|
||||
# Clip embeddings models handle text/image separately, so we don't need to pad input ids
|
||||
return input_ids
|
||||
|
||||
|
||||
@@ -5,6 +5,7 @@
|
||||
"""Inference-only Cohere2Vision (Command-A-Vision) multimodal model."""
|
||||
|
||||
import math
|
||||
from array import array
|
||||
from typing import Iterable, List, Optional, Tuple
|
||||
|
||||
import torch
|
||||
@@ -151,9 +152,7 @@ class Cohere2VisionForConditionalGeneration(nn.Module):
|
||||
# walks ``model.model.layers``) can locate the transformer layers.
|
||||
self.model = self.language_model.model
|
||||
|
||||
def pad_input_ids(
|
||||
self, input_ids: List[int], mm_inputs: MultimodalInputs
|
||||
) -> List[int]:
|
||||
def pad_input_ids(self, input_ids: array, mm_inputs: MultimodalInputs) -> array:
|
||||
pattern = MultiModalityDataPaddingPatternMultimodalTokens()
|
||||
return pattern.pad_input_tokens(input_ids, mm_inputs)
|
||||
|
||||
|
||||
@@ -19,6 +19,7 @@ Arcee causal-LM structure used by SGLang, while the vision path uses the native
|
||||
SigLIP2 implementation and an Edge-specific spatial-merge projector.
|
||||
"""
|
||||
|
||||
from array import array
|
||||
from typing import Iterable, List, Optional, Tuple
|
||||
|
||||
import numpy as np
|
||||
@@ -212,9 +213,7 @@ class Cosmos3EdgeForConditionalGeneration(ArceeForCausalLM):
|
||||
prefix=add_prefix("projector", prefix),
|
||||
)
|
||||
|
||||
def pad_input_ids(
|
||||
self, input_ids: List[int], mm_inputs: MultimodalInputs
|
||||
) -> List[int]:
|
||||
def pad_input_ids(self, input_ids: array, mm_inputs: MultimodalInputs) -> array:
|
||||
pattern = MultiModalityDataPaddingPatternMultimodalTokens()
|
||||
return pattern.pad_input_tokens(input_ids, mm_inputs)
|
||||
|
||||
|
||||
@@ -19,6 +19,7 @@
|
||||
import collections
|
||||
import math
|
||||
import os
|
||||
from array import array
|
||||
from dataclasses import field
|
||||
from enum import Enum
|
||||
from functools import partial
|
||||
@@ -1995,7 +1996,7 @@ class MultiModalityCausalLM(MultiModalityPreTrainedModel):
|
||||
def prepare_gen_img_embeds(self, image_ids: torch.LongTensor):
|
||||
return self.gen_aligner(self.gen_embed(image_ids))
|
||||
|
||||
def pad_input_ids(self, input_ids: List[int], image_inputs: MultimodalInputs):
|
||||
def pad_input_ids(self, input_ids: array, image_inputs: MultimodalInputs) -> array:
|
||||
im_start_id = image_inputs.im_start_id
|
||||
im_end_id = image_inputs.im_end_id
|
||||
media_token_pairs = [(im_start_id, im_end_id)]
|
||||
|
||||
@@ -22,6 +22,7 @@
|
||||
import copy
|
||||
import logging
|
||||
import math
|
||||
from array import array
|
||||
from functools import partial
|
||||
from typing import Iterable, List, Optional, Set, Tuple, Type, TypeAlias, Union
|
||||
|
||||
@@ -1763,7 +1764,7 @@ class DeepseekOCRForCausalLM(nn.Module):
|
||||
|
||||
return inputs_embeds
|
||||
|
||||
def pad_input_ids(self, input_ids: List[int], mm_inputs: MultimodalInputs):
|
||||
def pad_input_ids(self, input_ids: array, mm_inputs: MultimodalInputs) -> array:
|
||||
pattern = MultiModalityDataPaddingPatternMultimodalTokens()
|
||||
return pattern.pad_input_tokens(input_ids, mm_inputs)
|
||||
|
||||
|
||||
@@ -4,6 +4,7 @@ import concurrent.futures
|
||||
import functools
|
||||
import logging
|
||||
import time
|
||||
from array import array
|
||||
from contextlib import contextmanager, nullcontext
|
||||
from types import SimpleNamespace
|
||||
from typing import (
|
||||
@@ -4981,7 +4982,7 @@ class DeepseekV4ForCausalLM(nn.Module):
|
||||
def routed_experts_weights_of_layer(self):
|
||||
return self._routed_experts_weights_of_layer.value
|
||||
|
||||
def pad_input_ids(self, input_ids, mm_inputs):
|
||||
def pad_input_ids(self, input_ids: array, mm_inputs) -> array:
|
||||
return MultiModalityDataPaddingPatternMultimodalTokens().pad_input_tokens(
|
||||
input_ids, mm_inputs
|
||||
)
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
from array import array
|
||||
from typing import Iterable, List, Optional, Tuple
|
||||
|
||||
import torch
|
||||
@@ -262,7 +263,7 @@ class DeepseekVL2ForCausalLM(nn.Module):
|
||||
weights_loader = getattr(param, "weight_loader", default_weight_loader)
|
||||
weights_loader(param, loaded_weight)
|
||||
|
||||
def pad_input_ids(self, input_ids: List[int], mm_inputs: MultimodalInputs):
|
||||
def pad_input_ids(self, input_ids: array, mm_inputs: MultimodalInputs) -> array:
|
||||
pattern = MultiModalityDataPaddingPatternMultimodalTokens()
|
||||
return pattern.pad_input_tokens(input_ids, mm_inputs)
|
||||
|
||||
|
||||
@@ -19,6 +19,7 @@
|
||||
import concurrent.futures
|
||||
import logging
|
||||
import math
|
||||
from array import array
|
||||
from dataclasses import dataclass
|
||||
from enum import IntEnum, auto
|
||||
from typing import (
|
||||
@@ -1921,10 +1922,10 @@ class Dots3LanguageModelForCausalLM(nn.Module):
|
||||
|
||||
def pad_input_ids(
|
||||
self,
|
||||
input_ids: List[int],
|
||||
input_ids: array,
|
||||
mm_inputs: MultimodalInputs,
|
||||
**kwargs,
|
||||
) -> List[int]:
|
||||
) -> array:
|
||||
token_pairs = []
|
||||
if mm_inputs.im_start_id is not None and mm_inputs.im_end_id is not None:
|
||||
token_pairs.append((mm_inputs.im_start_id, mm_inputs.im_end_id))
|
||||
@@ -2637,7 +2638,7 @@ class DotsNoteOmniThinkerForConditionalGeneration(nn.Module):
|
||||
def get_input_embeddings(self):
|
||||
return self.language_model.get_input_embeddings()
|
||||
|
||||
def pad_input_ids(self, input_ids, mm_inputs, **kwargs):
|
||||
def pad_input_ids(self, input_ids: array, mm_inputs, **kwargs) -> array:
|
||||
return self.language_model.pad_input_ids(input_ids, mm_inputs, **kwargs)
|
||||
|
||||
def get_image_feature(self, items: List[MultimodalDataItem]) -> torch.Tensor:
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
# Adapted from Qwen2.5-VL SGLang implementation
|
||||
|
||||
import logging
|
||||
from array import array
|
||||
from typing import Iterable, List, Optional, Tuple
|
||||
|
||||
import torch
|
||||
@@ -56,7 +57,7 @@ class DotsOCRForCausalLM(nn.Module):
|
||||
|
||||
self.logits_processor = LogitsProcessor(config)
|
||||
|
||||
def pad_input_ids(self, input_ids: List[int], mm_inputs: MultimodalInputs):
|
||||
def pad_input_ids(self, input_ids: array, mm_inputs: MultimodalInputs) -> array:
|
||||
pattern = MultiModalityDataPaddingPatternMultimodalTokens()
|
||||
return pattern.pad_input_tokens(input_ids, mm_inputs)
|
||||
|
||||
|
||||
@@ -17,6 +17,7 @@
|
||||
# limitations under the License.
|
||||
"""Inference-only Dots-VL model compatible with HuggingFace weights."""
|
||||
|
||||
from array import array
|
||||
from typing import Iterable, List, Optional, Tuple
|
||||
|
||||
import torch
|
||||
@@ -132,7 +133,7 @@ class DotsVLMForCausalLM(nn.Module):
|
||||
def get_model_config_for_expert_location(cls, config):
|
||||
return DeepseekV2ForCausalLM.get_model_config_for_expert_location(config)
|
||||
|
||||
def pad_input_ids(self, input_ids: List[int], mm_inputs: MultimodalInputs):
|
||||
def pad_input_ids(self, input_ids: array, mm_inputs: MultimodalInputs) -> array:
|
||||
"""Pad input_ids with multimodal tokens"""
|
||||
# Get image token ID for padding pattern
|
||||
pattern = MultiModalityDataPaddingPatternMultimodalTokens()
|
||||
|
||||
@@ -13,6 +13,7 @@
|
||||
"""Inference-only Ernie45-VL model compatible with HuggingFace weights."""
|
||||
|
||||
import logging
|
||||
from array import array
|
||||
from functools import lru_cache, partial
|
||||
from typing import Iterable, List, Optional, Tuple, Type
|
||||
|
||||
@@ -588,7 +589,7 @@ class Ernie4_5_VLMoeForConditionalGeneration(nn.Module):
|
||||
else:
|
||||
self._visual_token_ids_tensor_cache = None
|
||||
|
||||
def pad_input_ids(self, input_ids: List[int], mm_inputs: MultimodalInputs):
|
||||
def pad_input_ids(self, input_ids: array, mm_inputs: MultimodalInputs) -> array:
|
||||
pattern = MultiModalityDataPaddingPatternMultimodalTokens()
|
||||
return pattern.pad_input_tokens(input_ids, mm_inputs)
|
||||
|
||||
|
||||
@@ -19,6 +19,7 @@
|
||||
|
||||
import logging
|
||||
import re
|
||||
from array import array
|
||||
from functools import lru_cache
|
||||
from typing import Iterable, List, Optional, Set, Tuple, TypedDict
|
||||
|
||||
@@ -201,9 +202,7 @@ class Gemma3ForConditionalGeneration(PreTrainedModel):
|
||||
self.language_model.logits_processor.logit_scale *= logit_scale
|
||||
self.post_init()
|
||||
|
||||
def pad_input_ids(
|
||||
self, input_ids: List[int], image_inputs: MultimodalInputs
|
||||
) -> List[int]:
|
||||
def pad_input_ids(self, input_ids: array, image_inputs: MultimodalInputs) -> array:
|
||||
"""Pad input IDs with image tokens."""
|
||||
# Get special token IDs
|
||||
im_start_id: int = image_inputs.im_start_id
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import logging
|
||||
import re
|
||||
from array import array
|
||||
from functools import lru_cache
|
||||
from typing import Iterable, List, Optional, Set, Tuple, TypedDict, Union
|
||||
|
||||
@@ -243,9 +244,9 @@ class Gemma3nForConditionalGeneration(PreTrainedModel):
|
||||
|
||||
def pad_input_ids(
|
||||
self,
|
||||
input_ids: List[int],
|
||||
input_ids: array,
|
||||
mm_inputs: MultimodalInputs,
|
||||
) -> List[int]:
|
||||
) -> array:
|
||||
"""Pad input IDs with image and audio tokens."""
|
||||
pattern = MultiModalityDataPaddingPatternMultimodalTokens()
|
||||
return pattern.pad_input_tokens(input_ids, mm_inputs)
|
||||
|
||||
@@ -15,6 +15,7 @@
|
||||
|
||||
import logging
|
||||
import re
|
||||
from array import array
|
||||
from functools import lru_cache
|
||||
from typing import Iterable, List, Optional, Set, Tuple, TypedDict, Union
|
||||
|
||||
@@ -296,9 +297,9 @@ class Gemma4ForConditionalGeneration(PreTrainedModel):
|
||||
|
||||
def pad_input_ids(
|
||||
self,
|
||||
input_ids: List[int],
|
||||
input_ids: array,
|
||||
mm_inputs: MultimodalInputs,
|
||||
) -> List[int]:
|
||||
) -> array:
|
||||
"""Pad input IDs with image and audio tokens."""
|
||||
pattern = MultiModalityDataPaddingPatternMultimodalTokens()
|
||||
return pattern.pad_input_tokens(input_ids, mm_inputs)
|
||||
|
||||
@@ -18,6 +18,7 @@
|
||||
"""Inference-only GLM-4.1V model compatible with HuggingFace weights."""
|
||||
|
||||
import logging
|
||||
from array import array
|
||||
from functools import lru_cache
|
||||
from typing import Iterable, List, Optional, Tuple
|
||||
|
||||
@@ -594,7 +595,7 @@ class Glm4vForConditionalGeneration(nn.Module):
|
||||
# For EAGLE3 support
|
||||
self.capture_aux_hidden_states = False
|
||||
|
||||
def pad_input_ids(self, input_ids: List[int], mm_inputs: MultimodalInputs):
|
||||
def pad_input_ids(self, input_ids: array, mm_inputs: MultimodalInputs) -> array:
|
||||
pattern = MultiModalityDataPaddingPatternMultimodalTokens()
|
||||
return pattern.pad_input_tokens(input_ids, mm_inputs)
|
||||
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import logging
|
||||
from array import array
|
||||
from contextlib import nullcontext
|
||||
from functools import partial
|
||||
from typing import Any, Dict, Iterable, List, Optional, Tuple, Union
|
||||
@@ -1419,7 +1420,7 @@ class Glm5NextForConditionalGeneration(nn.Module):
|
||||
# Capturing before layer k + 1 gives the completed output of layer k.
|
||||
self.model.layers_to_capture = [val + 1 for val in layer_ids]
|
||||
|
||||
def pad_input_ids(self, input_ids: List[int], mm_inputs: MultimodalInputs):
|
||||
def pad_input_ids(self, input_ids: array, mm_inputs: MultimodalInputs) -> array:
|
||||
pattern = MultiModalityDataPaddingPatternMultimodalTokens()
|
||||
return pattern.pad_input_tokens(input_ids, mm_inputs)
|
||||
|
||||
|
||||
@@ -20,6 +20,7 @@
|
||||
|
||||
import copy
|
||||
import logging
|
||||
from array import array
|
||||
from typing import Any, Dict, Iterable, List, Optional, Tuple
|
||||
|
||||
import torch
|
||||
@@ -1068,7 +1069,7 @@ class GlmImageForConditionalGeneration(nn.Module):
|
||||
|
||||
self.logits_processor = LogitsProcessor(logits_config)
|
||||
|
||||
def pad_input_ids(self, input_ids: List[int], mm_inputs: MultimodalInputs):
|
||||
def pad_input_ids(self, input_ids: array, mm_inputs: MultimodalInputs) -> array:
|
||||
pattern = MultiModalityDataPaddingPatternMultimodalTokens()
|
||||
return pattern.pad_input_tokens(input_ids, mm_inputs)
|
||||
|
||||
|
||||
@@ -18,6 +18,7 @@
|
||||
"""Inference-only GLM-ASR-HF model compatible with HuggingFace weights."""
|
||||
|
||||
import logging
|
||||
from array import array
|
||||
from typing import Any, Iterable, List, Optional, Tuple
|
||||
|
||||
import torch
|
||||
@@ -88,7 +89,7 @@ class GlmAsrForConditionalGeneration(nn.Module):
|
||||
)
|
||||
self.pattern = MultiModalityDataPaddingPatternMultimodalTokens()
|
||||
|
||||
def pad_input_ids(self, input_ids: List[int], mm_inputs: MultimodalInputs):
|
||||
def pad_input_ids(self, input_ids: array, mm_inputs: MultimodalInputs) -> array:
|
||||
return self.pattern.pad_input_tokens(input_ids, mm_inputs)
|
||||
|
||||
def get_audio_feature(self, items: List[MultimodalDataItem]) -> torch.Tensor:
|
||||
|
||||
@@ -3,6 +3,7 @@ from __future__ import annotations
|
||||
import copy
|
||||
import logging
|
||||
import re
|
||||
from array import array
|
||||
from typing import Iterable, Optional, Set, Tuple
|
||||
|
||||
import torch
|
||||
@@ -1122,7 +1123,7 @@ class InklingForConditionalGeneration(nn.Module):
|
||||
return 2
|
||||
return 1
|
||||
|
||||
def pad_input_ids(self, input_ids: list[int], mm_inputs: MultimodalInputs):
|
||||
def pad_input_ids(self, input_ids: array, mm_inputs: MultimodalInputs) -> array:
|
||||
# The processor expands one placeholder per media item into a run of the same
|
||||
# token id; the scheduler calls this to replace each run with the item's
|
||||
# pad_value (radix hash), which _embed_mm then masks on to scatter the embeds.
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
from array import array
|
||||
from typing import Iterable, List, Optional, Tuple
|
||||
|
||||
import torch
|
||||
@@ -154,7 +155,7 @@ class InternS1ForConditionalGeneration(nn.Module):
|
||||
|
||||
return hs
|
||||
|
||||
def pad_input_ids(self, input_ids: List[int], mm_inputs: MultimodalInputs):
|
||||
def pad_input_ids(self, input_ids: array, mm_inputs: MultimodalInputs) -> array:
|
||||
# Get all special token IDs
|
||||
im_start_id: int = mm_inputs.im_start_id
|
||||
im_end_id: int = mm_inputs.im_end_id
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from array import array
|
||||
from typing import Iterable, List, Optional, Tuple, Union
|
||||
|
||||
import torch
|
||||
@@ -674,7 +675,7 @@ class InternVLChatModel(nn.Module):
|
||||
|
||||
return hidden_states
|
||||
|
||||
def pad_input_ids(self, input_ids: List[int], mm_inputs: MultimodalInputs):
|
||||
def pad_input_ids(self, input_ids: array, mm_inputs: MultimodalInputs) -> array:
|
||||
# Get all special token IDs
|
||||
im_start_id: int = mm_inputs.im_start_id
|
||||
im_end_id: int = mm_inputs.im_end_id
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import math
|
||||
from array import array
|
||||
from collections.abc import Iterable
|
||||
|
||||
import einops
|
||||
@@ -133,9 +134,7 @@ class JetVLMForConditionalGeneration(nn.Module):
|
||||
)
|
||||
weight_loader(param, loaded_weight)
|
||||
|
||||
def pad_input_ids(
|
||||
self, input_ids: list[int], mm_inputs: MultimodalInputs
|
||||
) -> list[int]:
|
||||
def pad_input_ids(self, input_ids: array, mm_inputs: MultimodalInputs) -> array:
|
||||
pattern = MultiModalityDataPaddingPatternMultimodalTokens()
|
||||
return pattern.pad_input_tokens(input_ids, mm_inputs)
|
||||
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import logging
|
||||
from array import array
|
||||
from copy import deepcopy
|
||||
from typing import Iterable, List, Optional, Sequence, Tuple
|
||||
|
||||
@@ -775,7 +776,7 @@ class KimiK25ForConditionalGeneration(nn.Module):
|
||||
image_embeds = self.vision_tower(pixel_values, grid_thws)
|
||||
return mm_projection_auto(self.mm_projector, image_embeds)
|
||||
|
||||
def pad_input_ids(self, input_ids: List[int], mm_inputs: MultimodalInputs):
|
||||
def pad_input_ids(self, input_ids: array, mm_inputs: MultimodalInputs) -> array:
|
||||
pattern = MultiModalityDataPaddingPatternMultimodalTokens()
|
||||
return pattern.pad_input_tokens(input_ids, mm_inputs)
|
||||
|
||||
|
||||
@@ -9,6 +9,7 @@
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
from array import array
|
||||
from collections.abc import Iterable
|
||||
from functools import cached_property
|
||||
from types import SimpleNamespace
|
||||
@@ -3815,7 +3816,7 @@ class KimiK3ForConditionalGeneration(nn.Module):
|
||||
image_embeds = self.vision_tower(pixel_values, grid_thws_host.to(device))
|
||||
return self.mm_projector(image_embeds)
|
||||
|
||||
def pad_input_ids(self, input_ids: List[int], mm_inputs: MultimodalInputs):
|
||||
def pad_input_ids(self, input_ids: array, mm_inputs: MultimodalInputs) -> array:
|
||||
pattern = MultiModalityDataPaddingPatternMultimodalTokens()
|
||||
return pattern.pad_input_tokens(input_ids, mm_inputs)
|
||||
|
||||
|
||||
@@ -44,6 +44,7 @@
|
||||
import copy
|
||||
import logging
|
||||
import math
|
||||
from array import array
|
||||
from dataclasses import dataclass
|
||||
from typing import Iterable, List, Optional, Tuple
|
||||
|
||||
@@ -192,7 +193,7 @@ class KimiVLForConditionalGeneration(nn.Module):
|
||||
res = self.multi_modal_projector(image_features)
|
||||
return res
|
||||
|
||||
def pad_input_ids(self, input_ids: List[int], mm_inputs: MultimodalInputs):
|
||||
def pad_input_ids(self, input_ids: array, mm_inputs: MultimodalInputs) -> array:
|
||||
pattern = MultiModalityDataPaddingPatternMultimodalTokens()
|
||||
return pattern.pad_input_tokens(input_ids, mm_inputs)
|
||||
|
||||
|
||||
@@ -20,6 +20,7 @@ LFM2-VL is a vision-language model that combines:
|
||||
"""
|
||||
|
||||
import logging
|
||||
from array import array
|
||||
from typing import Iterable, List, Optional, Tuple
|
||||
|
||||
import numpy as np
|
||||
@@ -170,9 +171,7 @@ class Lfm2VlForConditionalGeneration(nn.Module):
|
||||
|
||||
self.logits_processor = LogitsProcessor(config.text_config)
|
||||
|
||||
def pad_input_ids(
|
||||
self, input_ids: List[int], mm_inputs: MultimodalInputs
|
||||
) -> List[int]:
|
||||
def pad_input_ids(self, input_ids: array, mm_inputs: MultimodalInputs) -> array:
|
||||
pattern = MultiModalityDataPaddingPatternMultimodalTokens()
|
||||
result = pattern.pad_input_tokens(input_ids, mm_inputs)
|
||||
return result
|
||||
|
||||
@@ -31,6 +31,7 @@ References:
|
||||
- https://huggingface.co/lightonai/LightOnOCR-2-1B
|
||||
"""
|
||||
|
||||
from array import array
|
||||
from dataclasses import fields
|
||||
from typing import Iterable, List, Tuple
|
||||
|
||||
@@ -125,7 +126,7 @@ class LightOnOCRForConditionalGeneration(nn.Module):
|
||||
quant_config=quant_config,
|
||||
)
|
||||
|
||||
def pad_input_ids(self, input_ids: List[int], mm_inputs: MultimodalInputs):
|
||||
def pad_input_ids(self, input_ids: array, mm_inputs: MultimodalInputs) -> array:
|
||||
pattern = MultiModalityDataPaddingPatternMultimodalTokens()
|
||||
return pattern.pad_input_tokens(input_ids, mm_inputs)
|
||||
|
||||
|
||||
@@ -650,7 +650,7 @@ class LlavaForConditionalGeneration(LlavaBaseForCausalLM):
|
||||
def dtype(self):
|
||||
return self.torch_dtype
|
||||
|
||||
def pad_input_ids(self, input_ids: List[int], image_inputs: MultimodalInputs):
|
||||
def pad_input_ids(self, input_ids: array, image_inputs: MultimodalInputs) -> array:
|
||||
if hasattr(self.vision_tower, "pad_input_ids"):
|
||||
return self.vision_tower.pad_input_ids(input_ids, image_inputs)
|
||||
else:
|
||||
|
||||
@@ -19,6 +19,7 @@ pattern.
|
||||
"""
|
||||
|
||||
import logging
|
||||
from array import array
|
||||
from typing import Any, Dict, Iterable, List, Optional, Set, Tuple
|
||||
|
||||
import torch
|
||||
@@ -132,7 +133,7 @@ class LocateAnythingForConditionalGeneration(nn.Module):
|
||||
assert isinstance(image_features, list)
|
||||
return self.multi_modal_projector(torch.cat(image_features))
|
||||
|
||||
def pad_input_ids(self, input_ids: List[int], mm_inputs: MultimodalInputs):
|
||||
def pad_input_ids(self, input_ids: array, mm_inputs: MultimodalInputs) -> array:
|
||||
pattern = MultiModalityDataPaddingPatternMultimodalTokens()
|
||||
return pattern.pad_input_tokens(input_ids, mm_inputs)
|
||||
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import collections
|
||||
import collections.abc
|
||||
import logging
|
||||
from array import array
|
||||
from collections.abc import Callable, Sequence
|
||||
from typing import Iterable, List, Optional, Tuple, TypeAlias, cast
|
||||
|
||||
@@ -502,7 +503,7 @@ class MiDashengLMModel(nn.Module):
|
||||
self.logits_processor = self.language_model.logits_processor
|
||||
self.quant_config = quant_config
|
||||
|
||||
def pad_input_ids(self, input_ids: List[int], mm_inputs: MultimodalInputs):
|
||||
def pad_input_ids(self, input_ids: array, mm_inputs: MultimodalInputs) -> array:
|
||||
"""Pad input IDs with multimodal tokens."""
|
||||
pattern = MultiModalityDataPaddingPatternMultimodalTokens()
|
||||
return pattern.pad_input_tokens(input_ids, mm_inputs)
|
||||
|
||||
@@ -15,6 +15,7 @@
|
||||
import logging
|
||||
import math
|
||||
import re
|
||||
from array import array
|
||||
from typing import Any, Dict, Iterable, List, Optional, Tuple, Union
|
||||
|
||||
import torch
|
||||
@@ -1273,7 +1274,7 @@ class MiMoV2ForCausalLM(nn.Module, AudioEncoderMixin):
|
||||
)
|
||||
return self.model.get_input_embedding(input_ids)
|
||||
|
||||
def pad_input_ids(self, input_ids: List[int], mm_inputs: MultimodalInputs):
|
||||
def pad_input_ids(self, input_ids: array, mm_inputs: MultimodalInputs) -> array:
|
||||
pattern = MultiModalityDataPaddingPatternMultimodalTokens()
|
||||
return pattern.pad_input_tokens(input_ids, mm_inputs)
|
||||
|
||||
|
||||
@@ -7,6 +7,7 @@ state_dict aligns 1:1 with ``self.named_parameters()``.
|
||||
"""
|
||||
|
||||
import logging
|
||||
from array import array
|
||||
from typing import Any, Iterable, List, Optional, Tuple
|
||||
|
||||
import torch
|
||||
@@ -56,7 +57,7 @@ class MiMoV2ASRForCausalLM(MiMoForCausalLM, AudioEncoderMixin):
|
||||
super().__init__(config, quant_config=quant_config, prefix=prefix)
|
||||
self.build_audio_encoder(MiMoAudioEncoderConfig(**config.audio_config))
|
||||
|
||||
def pad_input_ids(self, input_ids: List[int], mm_inputs: MultimodalInputs):
|
||||
def pad_input_ids(self, input_ids: array, mm_inputs: MultimodalInputs) -> array:
|
||||
pattern = MultiModalityDataPaddingPatternMultimodalTokens()
|
||||
return pattern.pad_input_tokens(input_ids, mm_inputs)
|
||||
|
||||
|
||||
@@ -16,6 +16,7 @@
|
||||
"""Inference-only MiniCPM-o model compatible with HuggingFace weights."""
|
||||
|
||||
import math
|
||||
from array import array
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, Iterable, List, Literal, Optional, Tuple, Union
|
||||
|
||||
@@ -1518,7 +1519,7 @@ class MiniCPMO(MiniCPMBaseModel):
|
||||
|
||||
return resampler.to(device=get_device(), dtype=torch.get_default_dtype())
|
||||
|
||||
def pad_input_ids(self, input_ids: List[int], mm_input: MultimodalInputs):
|
||||
def pad_input_ids(self, input_ids: array, mm_input: MultimodalInputs) -> array:
|
||||
# Get all special token IDs
|
||||
im_start_id: int = mm_input.im_start_id
|
||||
im_end_id: int = mm_input.im_end_id
|
||||
|
||||
@@ -22,6 +22,7 @@
|
||||
"""Inference-only MiniCPM-V model compatible with HuggingFace weights."""
|
||||
|
||||
import types
|
||||
from array import array
|
||||
from functools import partial
|
||||
from itertools import chain
|
||||
from typing import (
|
||||
@@ -983,7 +984,7 @@ class MiniCPMV2_6(MiniCPMBaseModel):
|
||||
)
|
||||
return self.resampler(vision_embedding, tgt_sizes)
|
||||
|
||||
def pad_input_ids(self, input_ids: List[int], image_inputs: MultimodalInputs):
|
||||
def pad_input_ids(self, input_ids: array, image_inputs: MultimodalInputs) -> array:
|
||||
# Get all special token IDs
|
||||
im_start_id: int = image_inputs.im_start_id
|
||||
im_end_id: int = image_inputs.im_end_id
|
||||
@@ -1149,7 +1150,7 @@ class MiniCPMV4_0(MiniCPMBaseModel):
|
||||
)
|
||||
return self.resampler(vision_embedding, tgt_sizes)
|
||||
|
||||
def pad_input_ids(self, input_ids: List[int], image_inputs: MultimodalInputs):
|
||||
def pad_input_ids(self, input_ids: array, image_inputs: MultimodalInputs) -> array:
|
||||
# Get all special token IDs
|
||||
im_start_id: int = image_inputs.im_start_id
|
||||
im_end_id: int = image_inputs.im_end_id
|
||||
@@ -1319,7 +1320,7 @@ class MiniCPMV4_5(MiniCPMBaseModel):
|
||||
)
|
||||
return self.resampler(vision_embedding, tgt_sizes)
|
||||
|
||||
def pad_input_ids(self, input_ids: List[int], image_inputs: MultimodalInputs):
|
||||
def pad_input_ids(self, input_ids: array, image_inputs: MultimodalInputs) -> array:
|
||||
# Get all special token IDs
|
||||
im_start_id: int = image_inputs.im_start_id
|
||||
im_end_id: int = image_inputs.im_end_id
|
||||
@@ -1544,7 +1545,7 @@ class MiniCPMV4_6(MiniCPMBaseModel):
|
||||
# source. sglang's dispatcher routes by ``get_{modality}_feature``.
|
||||
get_video_feature = get_image_feature
|
||||
|
||||
def pad_input_ids(self, input_ids: List[int], image_inputs: MultimodalInputs):
|
||||
def pad_input_ids(self, input_ids: array, image_inputs: MultimodalInputs) -> array:
|
||||
im_start_id: int = image_inputs.im_start_id
|
||||
im_end_id: int = image_inputs.im_end_id
|
||||
slice_start_id: int = image_inputs.slice_start_id
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
import logging
|
||||
from array import array
|
||||
from typing import Iterable, List, Optional, Tuple
|
||||
|
||||
import torch
|
||||
@@ -195,7 +196,7 @@ class MiniMaxM3SparseForConditionalGeneration(nn.Module):
|
||||
text_config
|
||||
)
|
||||
|
||||
def pad_input_ids(self, input_ids: List[int], mm_inputs: MultimodalInputs):
|
||||
def pad_input_ids(self, input_ids: array, mm_inputs: MultimodalInputs) -> array:
|
||||
return MultiModalityDataPaddingPatternMultimodalTokens().pad_input_tokens(
|
||||
input_ids, mm_inputs
|
||||
)
|
||||
|
||||
@@ -3,6 +3,7 @@ import logging
|
||||
import math
|
||||
import os
|
||||
import re
|
||||
from array import array
|
||||
from collections.abc import Iterable
|
||||
from typing import List, Optional, Set, Tuple
|
||||
|
||||
@@ -559,7 +560,7 @@ class Llama4ForConditionalGeneration(nn.Module):
|
||||
except (OSError, json_lib.JSONDecodeError, KeyError):
|
||||
return False
|
||||
|
||||
def pad_input_ids(self, input_ids: List[int], mm_inputs: MultimodalInputs):
|
||||
def pad_input_ids(self, input_ids: array, mm_inputs: MultimodalInputs) -> array:
|
||||
return self.padding_pattern.pad_input_tokens(input_ids, mm_inputs)
|
||||
|
||||
def get_image_feature(
|
||||
|
||||
@@ -13,6 +13,7 @@
|
||||
# ==============================================================================
|
||||
|
||||
import logging
|
||||
from array import array
|
||||
from typing import Iterable, List, Optional, Tuple
|
||||
|
||||
import torch
|
||||
@@ -977,7 +978,7 @@ class MuseGlimmerForConditionalGeneration(MuseGlimmerForCausalLM):
|
||||
config.hidden_size, eps=config.rms_norm_eps, has_weight=False
|
||||
)
|
||||
|
||||
def pad_input_ids(self, input_ids: List[int], mm_inputs: MultimodalInputs):
|
||||
def pad_input_ids(self, input_ids: array, mm_inputs: MultimodalInputs) -> array:
|
||||
return MultiModalityDataPaddingPatternMultimodalTokens().pad_input_tokens(
|
||||
input_ids, mm_inputs
|
||||
)
|
||||
|
||||
@@ -16,6 +16,7 @@
|
||||
# Adapted from https://github.com/vllm-project/vllm/blob/main/vllm/model_executor/models/nano_nemotron_vl.py
|
||||
|
||||
import logging
|
||||
from array import array
|
||||
from collections import deque
|
||||
from typing import Iterable
|
||||
|
||||
@@ -116,7 +117,7 @@ class NemotronH_Nano_VL_V2(EVS):
|
||||
|
||||
self.config = config
|
||||
|
||||
def pad_input_ids(self, input_ids: list[int], mm_inputs: MultimodalInputs):
|
||||
def pad_input_ids(self, input_ids: array, mm_inputs: MultimodalInputs) -> array:
|
||||
im_start_id: int = mm_inputs.im_start_id
|
||||
im_end_id: int = mm_inputs.im_end_id
|
||||
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import itertools
|
||||
import math
|
||||
from array import array
|
||||
from collections.abc import Iterable
|
||||
from typing import Any
|
||||
|
||||
@@ -218,9 +219,7 @@ class NVILAForConditionalGeneration(nn.Module):
|
||||
)
|
||||
weight_loader(param, loaded_weight)
|
||||
|
||||
def pad_input_ids(
|
||||
self, input_ids: list[int], mm_inputs: MultimodalInputs
|
||||
) -> list[int]:
|
||||
def pad_input_ids(self, input_ids: array, mm_inputs: MultimodalInputs) -> array:
|
||||
pattern = MultiModalityDataPaddingPatternMultimodalTokens()
|
||||
return pattern.pad_input_tokens(input_ids, mm_inputs)
|
||||
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import math
|
||||
from array import array
|
||||
from collections.abc import Iterable
|
||||
from typing import Any
|
||||
|
||||
@@ -178,9 +179,7 @@ class NVILALiteForConditionalGeneration(nn.Module):
|
||||
)
|
||||
weight_loader(param, loaded_weight)
|
||||
|
||||
def pad_input_ids(
|
||||
self, input_ids: list[int], mm_inputs: MultimodalInputs
|
||||
) -> list[int]:
|
||||
def pad_input_ids(self, input_ids: array, mm_inputs: MultimodalInputs) -> array:
|
||||
pattern = MultiModalityDataPaddingPatternMultimodalTokens()
|
||||
return pattern.pad_input_tokens(input_ids, mm_inputs)
|
||||
|
||||
|
||||
@@ -24,6 +24,7 @@ projections run once for the batch instead of once per image.
|
||||
"""
|
||||
|
||||
import itertools
|
||||
from array import array
|
||||
from collections.abc import Iterable
|
||||
from typing import List, Optional, Set, Tuple
|
||||
|
||||
@@ -584,7 +585,7 @@ class PaddleOCRVLForConditionalGeneration(Ernie4_5_ForCausalLM):
|
||||
)
|
||||
self.is_mrope_enabled = "mrope_section" in (self.config.rope_scaling or {})
|
||||
|
||||
def pad_input_ids(self, input_ids: List[int], mm_inputs: MultimodalInputs):
|
||||
def pad_input_ids(self, input_ids: array, mm_inputs: MultimodalInputs) -> array:
|
||||
pattern = MultiModalityDataPaddingPatternMultimodalTokens()
|
||||
return pattern.pad_input_tokens(input_ids, mm_inputs)
|
||||
|
||||
|
||||
@@ -20,6 +20,7 @@
|
||||
import logging
|
||||
import math
|
||||
import re
|
||||
from array import array
|
||||
from collections.abc import Iterable
|
||||
from typing import List, Optional, Tuple
|
||||
|
||||
@@ -475,7 +476,7 @@ class Phi4MMForCausalLM(nn.Module):
|
||||
|
||||
return hidden_states
|
||||
|
||||
def pad_input_ids(self, input_ids: List[int], mm_inputs: MultimodalInputs):
|
||||
def pad_input_ids(self, input_ids: array, mm_inputs: MultimodalInputs) -> array:
|
||||
pattern = MultiModalityDataPaddingPatternMultimodalTokens()
|
||||
return pattern.pad_input_tokens(input_ids, mm_inputs)
|
||||
|
||||
|
||||
@@ -16,6 +16,7 @@
|
||||
Using mistral-community/pixtral-12b as reference.
|
||||
"""
|
||||
|
||||
from array import array
|
||||
from dataclasses import dataclass, fields
|
||||
from typing import Iterable, List, Optional, Set, Tuple, Union
|
||||
|
||||
@@ -137,7 +138,7 @@ class PixtralForConditionalGeneration(nn.Module):
|
||||
self.vision_args, dim=self.config.text_config.hidden_size
|
||||
)
|
||||
|
||||
def pad_input_ids(self, input_ids: List[int], mm_inputs: MultimodalInputs):
|
||||
def pad_input_ids(self, input_ids: array, mm_inputs: MultimodalInputs) -> array:
|
||||
pattern = MultiModalityDataPaddingPatternMultimodalTokens()
|
||||
return pattern.pad_input_tokens(input_ids, mm_inputs)
|
||||
|
||||
@@ -858,7 +859,7 @@ class PixtralHFVisionModel(nn.Module):
|
||||
|
||||
DEFAULT_IMAGE_TOKEN_ID = 10
|
||||
|
||||
def pad_input_ids(self, input_ids: List[int], mm_inputs: MultimodalInputs):
|
||||
def pad_input_ids(self, input_ids: array, mm_inputs: MultimodalInputs) -> array:
|
||||
return self.input_padder.pad_input_tokens(input_ids, mm_inputs)
|
||||
|
||||
def __init__(
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import copy
|
||||
from array import array
|
||||
from typing import Iterable, List, Optional, Set, Tuple
|
||||
|
||||
import torch
|
||||
@@ -101,7 +102,7 @@ class POINTSV15ChatModel(nn.Module):
|
||||
prefix=add_prefix("vision_projector", prefix),
|
||||
)
|
||||
|
||||
def pad_input_ids(self, input_ids: List[int], mm_inputs: MultimodalInputs):
|
||||
def pad_input_ids(self, input_ids: array, mm_inputs: MultimodalInputs) -> array:
|
||||
pattern = MultiModalityDataPaddingPatternMultimodalTokens()
|
||||
return pattern.pad_input_tokens(input_ids, mm_inputs)
|
||||
|
||||
|
||||
@@ -25,6 +25,7 @@
|
||||
|
||||
import logging
|
||||
import re
|
||||
from array import array
|
||||
from functools import partial
|
||||
from typing import Iterable, List, Optional, Tuple, Type
|
||||
|
||||
@@ -767,7 +768,7 @@ class Qwen2_5_VLForConditionalGeneration(nn.Module):
|
||||
# For EAGLE3 support
|
||||
self.capture_aux_hidden_states = False
|
||||
|
||||
def pad_input_ids(self, input_ids: List[int], mm_inputs: MultimodalInputs):
|
||||
def pad_input_ids(self, input_ids: array, mm_inputs: MultimodalInputs) -> array:
|
||||
pattern = MultiModalityDataPaddingPatternMultimodalTokens()
|
||||
return pattern.pad_input_tokens(input_ids, mm_inputs)
|
||||
|
||||
|
||||
@@ -24,6 +24,7 @@
|
||||
"""Inference-only Qwen2-Audio model compatible with HuggingFace weights."""
|
||||
|
||||
import logging
|
||||
from array import array
|
||||
from typing import Any, Iterable, List, Optional, Tuple
|
||||
|
||||
import torch
|
||||
@@ -100,7 +101,7 @@ class Qwen2AudioForConditionalGeneration(nn.Module):
|
||||
)
|
||||
self.pattern = MultiModalityDataPaddingPatternMultimodalTokens()
|
||||
|
||||
def pad_input_ids(self, input_ids: List[int], mm_inputs: MultimodalInputs):
|
||||
def pad_input_ids(self, input_ids: array, mm_inputs: MultimodalInputs) -> array:
|
||||
return self.pattern.pad_input_tokens(input_ids, mm_inputs)
|
||||
|
||||
def get_audio_feature(self, items: List[MultimodalDataItem]) -> torch.Tensor:
|
||||
|
||||
@@ -24,6 +24,7 @@
|
||||
"""Inference-only Qwen2-VL model compatible with HuggingFace weights."""
|
||||
|
||||
import logging
|
||||
from array import array
|
||||
from functools import lru_cache, partial
|
||||
from typing import Iterable, List, Optional, Tuple, Type, TypedDict
|
||||
|
||||
@@ -492,7 +493,7 @@ class Qwen2VLForConditionalGeneration(nn.Module):
|
||||
self.logits_processor = LogitsProcessor(config)
|
||||
self.pooler = Pooler(pooling_type=PoolingType.LAST, normalize=True)
|
||||
|
||||
def pad_input_ids(self, input_ids: List[int], mm_inputs: MultimodalInputs):
|
||||
def pad_input_ids(self, input_ids: array, mm_inputs: MultimodalInputs) -> array:
|
||||
pattern = MultiModalityDataPaddingPatternMultimodalTokens()
|
||||
return pattern.pad_input_tokens(input_ids, mm_inputs)
|
||||
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
"""Qwen3-ASR model compatible with HuggingFace weights"""
|
||||
|
||||
import logging
|
||||
from array import array
|
||||
from typing import Any, Iterable, List, Optional, Tuple
|
||||
|
||||
import torch
|
||||
@@ -66,7 +67,7 @@ class Qwen3ASRForConditionalGeneration(nn.Module):
|
||||
)
|
||||
self.pattern = MultiModalityDataPaddingPatternMultimodalTokens()
|
||||
|
||||
def pad_input_ids(self, input_ids: List[int], mm_inputs: MultimodalInputs):
|
||||
def pad_input_ids(self, input_ids: array, mm_inputs: MultimodalInputs) -> array:
|
||||
return self.pattern.pad_input_tokens(input_ids, mm_inputs)
|
||||
|
||||
def get_audio_feature(self, items: List[MultimodalDataItem]) -> torch.Tensor:
|
||||
|
||||
@@ -16,6 +16,7 @@
|
||||
|
||||
import logging
|
||||
import re
|
||||
from array import array
|
||||
from collections import defaultdict
|
||||
from functools import lru_cache, partial
|
||||
from typing import Callable, Iterable, List, Optional, Tuple, Union
|
||||
@@ -1420,7 +1421,7 @@ class Qwen3VLForConditionalGeneration(nn.Module):
|
||||
cfg = getattr(model, "config", None)
|
||||
return int(getattr(cfg, "num_hidden_layers", 0))
|
||||
|
||||
def pad_input_ids(self, input_ids: List[int], mm_inputs: MultimodalInputs):
|
||||
def pad_input_ids(self, input_ids: array, mm_inputs: MultimodalInputs) -> array:
|
||||
if mm_inputs and mm_inputs.mm_items:
|
||||
_require_vision(self)
|
||||
pattern = MultiModalityDataPaddingPatternMultimodalTokens()
|
||||
|
||||
@@ -14,6 +14,7 @@
|
||||
"""Inference-only Sarashina2Vision model compatible with HuggingFace weights."""
|
||||
|
||||
import logging
|
||||
from array import array
|
||||
from typing import Iterable, List, Optional, Tuple
|
||||
|
||||
import torch
|
||||
@@ -107,7 +108,7 @@ class Sarashina2VisionForCausalLM(nn.Module):
|
||||
self.logits_processor = LogitsProcessor(config)
|
||||
self.pooler = Pooler(pooling_type=PoolingType.LAST, normalize=True)
|
||||
|
||||
def pad_input_ids(self, input_ids: List[int], mm_inputs: MultimodalInputs):
|
||||
def pad_input_ids(self, input_ids: array, mm_inputs: MultimodalInputs) -> array:
|
||||
"""Pad input tokens with multimodal data hashes for RadixAttention."""
|
||||
pattern = MultiModalityDataPaddingPatternMultimodalTokens()
|
||||
return pattern.pad_input_tokens(input_ids, mm_inputs)
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import logging
|
||||
import math
|
||||
from array import array
|
||||
from math import sqrt
|
||||
from typing import Any, Dict, Iterable, List, Optional, Tuple
|
||||
|
||||
@@ -895,7 +896,7 @@ class Step3VLForConditionalGeneration(nn.Module):
|
||||
)
|
||||
return self._flatten_embeddings(merged_image_features)
|
||||
|
||||
def pad_input_ids(self, input_ids: List[int], mm_inputs: MultimodalInputs):
|
||||
def pad_input_ids(self, input_ids: array, mm_inputs: MultimodalInputs) -> array:
|
||||
pattern = MultiModalityDataPaddingPatternMultimodalTokens()
|
||||
return pattern.pad_input_tokens(input_ids, mm_inputs)
|
||||
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""This is basically a copy from perception_models/core/vision_encoder/pe.py"""
|
||||
|
||||
from array import array
|
||||
from functools import partial
|
||||
from typing import Callable, Iterable, List, Optional, Tuple
|
||||
|
||||
@@ -570,7 +571,7 @@ class StepVLForConditionalGeneration(nn.Module):
|
||||
)
|
||||
return self._flatten_embeddings(merged_image_features)
|
||||
|
||||
def pad_input_ids(self, input_ids: List[int], mm_inputs: MultimodalInputs):
|
||||
def pad_input_ids(self, input_ids: array, mm_inputs: MultimodalInputs) -> array:
|
||||
pattern = MultiModalityDataPaddingPatternMultimodalTokens()
|
||||
return pattern.pad_input_tokens(input_ids, mm_inputs)
|
||||
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
from array import array
|
||||
from typing import Iterable, List, Optional, Tuple
|
||||
|
||||
import torch
|
||||
@@ -128,7 +129,7 @@ class Step3p7ForConditionalGeneration(nn.Module):
|
||||
)
|
||||
return self._flatten_embeddings(merged_image_features)
|
||||
|
||||
def pad_input_ids(self, input_ids: List[int], mm_inputs: MultimodalInputs):
|
||||
def pad_input_ids(self, input_ids: array, mm_inputs: MultimodalInputs) -> array:
|
||||
pattern = MultiModalityDataPaddingPatternMultimodalTokens()
|
||||
return pattern.pad_input_tokens(input_ids, mm_inputs)
|
||||
|
||||
|
||||
@@ -21,6 +21,7 @@
|
||||
import inspect
|
||||
import logging
|
||||
import re
|
||||
from array import array
|
||||
from collections.abc import Iterable, Mapping
|
||||
from contextlib import contextmanager
|
||||
from typing import List, Literal, Optional, Tuple, Union
|
||||
@@ -1368,7 +1369,7 @@ class MultiModalMixin:
|
||||
rope_type = str(getattr(self.text_config, "rope_type", "")).lower()
|
||||
return "mrope" in rope_type
|
||||
|
||||
def pad_input_ids(self, input_ids: list[int], mm_inputs: MultimodalInputs):
|
||||
def pad_input_ids(self, input_ids: array, mm_inputs: MultimodalInputs) -> array:
|
||||
return input_ids
|
||||
|
||||
def _get_modality_encoder(self, modality_name: str):
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
"""Standalone UNLIMITED-OCR model (SAM + CLIP vision encoders, Deepseek backbone)."""
|
||||
|
||||
import logging
|
||||
from array import array
|
||||
from typing import Iterable, List, Optional, Set, Tuple, TypeAlias, Union
|
||||
|
||||
import torch
|
||||
@@ -311,7 +312,7 @@ class UnlimitedOCRForCausalLM(nn.Module):
|
||||
)
|
||||
return inputs_embeds
|
||||
|
||||
def pad_input_ids(self, input_ids: List[int], mm_inputs: MultimodalInputs):
|
||||
def pad_input_ids(self, input_ids: array, mm_inputs: MultimodalInputs) -> array:
|
||||
"""Pad input token IDs with multimodal placeholder tokens."""
|
||||
pattern = MultiModalityDataPaddingPatternMultimodalTokens()
|
||||
return pattern.pad_input_tokens(input_ids, mm_inputs)
|
||||
|
||||
@@ -9,6 +9,7 @@
|
||||
"""Inference-only Voxtral (speech-to-text) model."""
|
||||
|
||||
import math
|
||||
from array import array
|
||||
from typing import Any, Iterable, List, Optional, Tuple
|
||||
|
||||
import torch
|
||||
@@ -337,7 +338,7 @@ class VoxtralForConditionalGeneration(nn.Module):
|
||||
|
||||
return results
|
||||
|
||||
def pad_input_ids(self, input_ids: List[int], mm_inputs: MultimodalInputs):
|
||||
def pad_input_ids(self, input_ids: array, mm_inputs: MultimodalInputs) -> array:
|
||||
return self.pattern.pad_input_tokens(input_ids, mm_inputs)
|
||||
|
||||
def get_audio_feature(self, items: List[MultimodalDataItem]) -> torch.Tensor:
|
||||
|
||||
@@ -15,6 +15,8 @@
|
||||
# ==============================================================================
|
||||
# Adapted from https://github.com/vllm-project/vllm/blob/main/vllm/multimodal/evs.py
|
||||
|
||||
from array import array
|
||||
|
||||
import torch
|
||||
|
||||
|
||||
@@ -122,8 +124,7 @@ def tokens_per_frame(
|
||||
|
||||
|
||||
def replace_offsets_with_tokens_per_frame(
|
||||
*,
|
||||
pre_chunked_input_ids: list[int],
|
||||
pre_chunked_input_ids: list[int] | array,
|
||||
num_tokens_per_frame: list[int],
|
||||
frame_offsets_inclusive: list[tuple[int, int]],
|
||||
filler_token_id: int,
|
||||
@@ -157,12 +158,13 @@ def replace_offsets_with_tokens_per_frame(
|
||||
... filler_token_id=0,
|
||||
... ) == [0, 1, 4, 0, 0, 0, 0, 5, 9, 10]
|
||||
"""
|
||||
assert isinstance(pre_chunked_input_ids, list)
|
||||
# Encoder metadata uses lists; model padding can replace it with packed tokens.
|
||||
assert isinstance(pre_chunked_input_ids, (list, array))
|
||||
ids = pre_chunked_input_ids
|
||||
|
||||
if len(frame_offsets_inclusive) == 1:
|
||||
"""There might be no frame separators, in which case there will be one contiguous span of tokens"""
|
||||
final = ids[0 : frame_offsets_inclusive[0][0]]
|
||||
final = list(ids[0 : frame_offsets_inclusive[0][0]])
|
||||
frames = [filler_token_id] * sum(num_tokens_per_frame)
|
||||
final.extend(frames)
|
||||
else:
|
||||
|
||||
@@ -1281,6 +1281,13 @@ class Range(NamedTuple):
|
||||
return self.end - self.start
|
||||
|
||||
|
||||
def assert_int64_array(values: array, name: str) -> None:
|
||||
"""Require a signed int64 array suitable for zero-copy tensor views."""
|
||||
assert (
|
||||
isinstance(values, array) and values.typecode == "q" and values.itemsize == 8
|
||||
), f"{name} must be array('q') with 8-byte items"
|
||||
|
||||
|
||||
def flatten_arrays_to_pinned_cpu(parts: List[array[int]], pin: bool) -> torch.Tensor:
|
||||
"""Flatten array.array('q') buffers into one int64 CPU tensor.
|
||||
|
||||
|
||||
Reference in New Issue
Block a user