[Fix] VLM: harden Kimi image cache identity across preprocessing and radix hashing (#43043)
Co-authored-by: Rahul Chalamala <22563365+rchalamala@users.noreply.github.com> Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Co-authored-by: Mick Qian <mickqian@users.noreply.github.com> Co-authored-by: Liangsheng Yin <hnyls2002@gmail.com>
This commit is contained in:
co-authored by
Rahul Chalamala
Devin AI
Mick Qian
Liangsheng Yin
parent
5a930cfd88
commit
3b932cde26
@@ -266,7 +266,7 @@ def pad_mm_token_pairs(
|
||||
if data_start_token_ids is None:
|
||||
data_start_token_ids = [s for s, _e in data_token_pairs]
|
||||
assert_int64_array(input_ids, "input_ids")
|
||||
pad_values = [item.pad_value for item in mm_inputs.mm_items]
|
||||
pad_items = mm_inputs.mm_items
|
||||
mm_inputs.data_offsets = []
|
||||
if data_token_pairs is None:
|
||||
data_token_pairs = [mm_inputs.im_start_id, mm_inputs.im_end_id]
|
||||
@@ -295,13 +295,12 @@ def pad_mm_token_pairs(
|
||||
data_idx += 1
|
||||
mm_inputs.data_offsets += [start_idx]
|
||||
|
||||
if data_idx >= len(pad_values):
|
||||
data_idx = len(pad_values) - 1
|
||||
if data_idx >= len(pad_items):
|
||||
data_idx = len(pad_items) - 1
|
||||
|
||||
num_tokens = end_idx - start_idx - 1
|
||||
pad_value = pad_values[data_idx]
|
||||
if num_tokens > 0:
|
||||
padded_ids.extend(array("q", [pad_value]) * num_tokens)
|
||||
padded_ids.extend(pad_items[data_idx].padding_sequence(num_tokens))
|
||||
|
||||
last_idx = end_idx
|
||||
|
||||
@@ -341,7 +340,10 @@ def pad_mm_tokens(input_ids: array, mm_inputs: MultimodalInputs) -> array:
|
||||
|
||||
for i, item in enumerate(items):
|
||||
for offset in items[i].offsets:
|
||||
input_ids_tensor[offset[0] : offset[1] + 1] = item.pad_value
|
||||
input_ids_tensor[offset[0] : offset[1] + 1] = torch.as_tensor(
|
||||
item.padding_sequence(offset[1] - offset[0] + 1),
|
||||
dtype=input_ids_tensor.dtype,
|
||||
)
|
||||
|
||||
return padded_input_ids
|
||||
|
||||
@@ -493,7 +495,7 @@ def embed_mm_inputs(
|
||||
embedder = getattr(multimodal_model, f"get_{modality_id}_feature", None)
|
||||
if len(items) != 0:
|
||||
assert embedder is not None, f"no embedding method found for {modality}"
|
||||
pad_values = [item.pad_value for item in items]
|
||||
pad_values = [value for item in items for value in item.padding_values()]
|
||||
if input_ids.device.type == "cuda":
|
||||
# Pinned staging keeps the placeholder copy asynchronous on CUDA.
|
||||
placeholder_cpu = torch.tensor(
|
||||
@@ -1189,6 +1191,8 @@ def get_new_expanded_mm_items(original_mm_items):
|
||||
# explicit None check, not `a or b`: the value is a multi-element
|
||||
# tensor whose truthiness is ambiguous.)
|
||||
image_grid_thw = item.model_specific_data.get("image_grid_thw")
|
||||
if image_grid_thw is None:
|
||||
image_grid_thw = item.model_specific_data.get("grid_thws")
|
||||
if image_grid_thw is None:
|
||||
image_grid_thw = item.model_specific_data.get("image_grid_hws")
|
||||
grid_len = _get_length(image_grid_thw)
|
||||
|
||||
@@ -53,6 +53,7 @@ ScheduleBatch -> ForwardBatch
|
||||
|
||||
import copy
|
||||
import dataclasses
|
||||
import hashlib
|
||||
import logging
|
||||
import re
|
||||
import sys
|
||||
@@ -160,6 +161,10 @@ INIT_INCREMENTAL_DETOKENIZATION_OFFSET = 5
|
||||
# This ensures pad_values don't overlap with valid text token IDs.
|
||||
MM_PAD_SHIFT_VALUE = 1_000_000
|
||||
_MM_HASH_MASK = (1 << 64) - 1
|
||||
_MM_IDENTITY_BYTES = 32
|
||||
_MM_WIDE_PAD_TAG = b"sglang-mm-wide-pad-v1"
|
||||
# Keep MM_PAD_SHIFT_VALUE + sentinel below 2**63 so it fits an int64 token ID.
|
||||
_MM_WIDE_PAD_MASK = (1 << 62) - 1
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -255,9 +260,15 @@ def split_cached_prefix_by_tier(
|
||||
return device, host, storage
|
||||
|
||||
|
||||
def _compute_pad_value(hash: int) -> int:
|
||||
"""Compute pad value from hash."""
|
||||
return MM_PAD_SHIFT_VALUE + (hash % (1 << 30))
|
||||
def _compute_pad_values(identity: bytes) -> Tuple[int, ...]:
|
||||
"""derive shared wide sentinels from an item hash or full identity digest"""
|
||||
digest = hashlib.sha256(_MM_WIDE_PAD_TAG + identity).digest()
|
||||
# prefix matching can stop after one sentinel, so each must retain wide bits
|
||||
return tuple(
|
||||
MM_PAD_SHIFT_VALUE
|
||||
+ (int.from_bytes(digest[i : i + 8], byteorder="big") & _MM_WIDE_PAD_MASK)
|
||||
for i in range(0, _MM_IDENTITY_BYTES, 8)
|
||||
)
|
||||
|
||||
|
||||
class BaseFinishReason:
|
||||
@@ -391,6 +402,11 @@ class MultimodalDataItem(msgspec.Struct, kw_only=True, dict=True, array_like=Tru
|
||||
default_factory=dict
|
||||
)
|
||||
|
||||
# optional full identity; otherwise sentinels derive from the 64-bit item hash
|
||||
# pad_value remains the first sentinel for scalar-padding consumers
|
||||
identity: Optional[bytes] = None
|
||||
pad_values: Optional[Tuple[int, ...]] = None
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
if self.hash is not None:
|
||||
msgspec.Struct.__setattr__(self, "hash", self.hash & _MM_HASH_MASK)
|
||||
@@ -419,7 +435,33 @@ class MultimodalDataItem(msgspec.Struct, kw_only=True, dict=True, array_like=Tru
|
||||
|
||||
def set_hash(self, hash_value: int) -> None:
|
||||
self.hash = hash_value
|
||||
self.pad_value = _compute_pad_value(hash_value)
|
||||
self.identity = None
|
||||
self.pad_values = _compute_pad_values(self.hash.to_bytes(8, byteorder="big"))
|
||||
self.pad_value = self.pad_values[0]
|
||||
|
||||
def set_identity(self, identity: bytes) -> None:
|
||||
"""Adopt a full 32-byte digest and derive four 62-bit pad sentinels.
|
||||
|
||||
``hash`` keeps the leading 64 bits for scalar consumers such as the
|
||||
embedding cache; the radix key uses the full sentinel pattern.
|
||||
"""
|
||||
if len(identity) != _MM_IDENTITY_BYTES:
|
||||
raise ValueError(
|
||||
f"multimodal identity must be {_MM_IDENTITY_BYTES} bytes, "
|
||||
f"got {len(identity)}"
|
||||
)
|
||||
self.identity = bytes(identity)
|
||||
self.hash = int.from_bytes(identity[:8], byteorder="big", signed=False)
|
||||
self.pad_values = _compute_pad_values(self.identity)
|
||||
self.pad_value = self.pad_values[0]
|
||||
|
||||
def padding_values(self) -> Tuple[int, ...]:
|
||||
return self.pad_values or (self.pad_value,)
|
||||
|
||||
def padding_sequence(self, length: int) -> List[int]:
|
||||
values = self.padding_values()
|
||||
repeats, remainder = divmod(length, len(values))
|
||||
return list(values) * repeats + list(values[:remainder])
|
||||
|
||||
@staticmethod
|
||||
def is_empty_list(l):
|
||||
@@ -433,12 +475,13 @@ class MultimodalDataItem(msgspec.Struct, kw_only=True, dict=True, array_like=Tru
|
||||
|
||||
from sglang.srt.multimodal.cache import resolve_multimodal_item_hash
|
||||
|
||||
self.hash = resolve_multimodal_item_hash(
|
||||
existing_hash=self.hash,
|
||||
feature=self.feature,
|
||||
precomputed_embeddings=self.precomputed_embeddings,
|
||||
self.set_hash(
|
||||
resolve_multimodal_item_hash(
|
||||
existing_hash=self.hash,
|
||||
feature=self.feature,
|
||||
precomputed_embeddings=self.precomputed_embeddings,
|
||||
)
|
||||
)
|
||||
self.pad_value = _compute_pad_value(self.hash)
|
||||
|
||||
def is_modality(self, modality: Modality) -> bool:
|
||||
return self.modality == modality
|
||||
@@ -650,7 +693,9 @@ class MultimodalProcessorOutput(
|
||||
|
||||
for item in mm_items:
|
||||
for start, end in item.offsets:
|
||||
padded_input_ids[start : end + 1] = [item.pad_value] * (end - start + 1)
|
||||
padded_input_ids[start : end + 1] = item.padding_sequence(
|
||||
end - start + 1
|
||||
)
|
||||
return padded_input_ids
|
||||
|
||||
@staticmethod
|
||||
|
||||
@@ -61,12 +61,18 @@ parse_prior_digest(py::object prior_digest_obj, bool *has_prior_digest) {
|
||||
|
||||
inline void hash_page(const unsigned char *data, std::size_t len,
|
||||
bool &has_prior_digest,
|
||||
std::array<unsigned char, kDigestLen> &prior_digest) {
|
||||
std::array<unsigned char, kDigestLen> &prior_digest,
|
||||
bool wide = false) {
|
||||
SHA256_CTX ctx;
|
||||
SHA256_Init(&ctx);
|
||||
if (has_prior_digest) {
|
||||
SHA256_Update(&ctx, prior_digest.data(), prior_digest.size());
|
||||
}
|
||||
if (wide) {
|
||||
// an odd byte count separates wide encoding from legacy u32 pages
|
||||
const unsigned char tag = 0xff;
|
||||
SHA256_Update(&ctx, &tag, sizeof(tag));
|
||||
}
|
||||
if (len > 0) {
|
||||
SHA256_Update(&ctx, data, len);
|
||||
}
|
||||
@@ -74,6 +80,40 @@ inline void hash_page(const unsigned char *data, std::size_t len,
|
||||
has_prior_digest = true;
|
||||
}
|
||||
|
||||
template <typename RawToken>
|
||||
bool has_wide_tokens(const RawToken *raw, std::size_t count) {
|
||||
if constexpr (sizeof(RawToken) == 4) {
|
||||
return false;
|
||||
}
|
||||
bool wide = false;
|
||||
for (std::size_t i = 0; i < count; ++i) {
|
||||
if (raw[i] > INT64_MAX) {
|
||||
throw std::out_of_range("token id must be a non-negative int64");
|
||||
}
|
||||
wide |= raw[i] > UINT32_MAX;
|
||||
}
|
||||
return wide;
|
||||
}
|
||||
|
||||
template <typename RawToken>
|
||||
void hash_wide_page(const RawToken *raw, std::size_t count, bool is_bigram,
|
||||
bool &has_prior_digest,
|
||||
std::array<unsigned char, kDigestLen> &prior_digest) {
|
||||
std::vector<std::uint64_t> words;
|
||||
if (is_bigram) {
|
||||
words.reserve(count * 2);
|
||||
for (std::size_t i = 0; i < count; ++i) {
|
||||
words.push_back(raw[i]);
|
||||
words.push_back(raw[i + 1]);
|
||||
}
|
||||
} else {
|
||||
words.assign(raw, raw + count);
|
||||
}
|
||||
hash_page(reinterpret_cast<const unsigned char *>(words.data()),
|
||||
words.size() * sizeof(std::uint64_t), has_prior_digest,
|
||||
prior_digest, true);
|
||||
}
|
||||
|
||||
template <typename RawToken>
|
||||
inline void fill_regular_page(const RawToken *raw, std::size_t start,
|
||||
std::size_t count, std::uint32_t *out) {
|
||||
@@ -203,6 +243,16 @@ void hash_pages_to_hex_blob(const RawToken *raw, std::size_t logical_len,
|
||||
for (std::size_t start = 0, page_idx = 0; start < logical_len;
|
||||
start += page_size, ++page_idx) {
|
||||
const std::size_t page_units = std::min(page_size, logical_len - start);
|
||||
const auto *page_start = raw + (is_bigram ? start : start * unit_width);
|
||||
const std::size_t raw_count =
|
||||
is_bigram ? page_units + 1 : page_units * unit_width;
|
||||
if (has_wide_tokens(page_start, raw_count)) {
|
||||
hash_wide_page(page_start, is_bigram ? page_units : raw_count, is_bigram,
|
||||
has_prior_digest, prior_digest);
|
||||
digest_to_hex_chars(prior_digest.data(),
|
||||
hex_blob.data() + page_idx * kHexLen);
|
||||
continue;
|
||||
}
|
||||
const std::size_t page_bytes =
|
||||
page_units * unit_width * sizeof(std::uint32_t);
|
||||
const unsigned char *bytes = nullptr;
|
||||
@@ -233,6 +283,13 @@ std::string hash_all(const RawToken *raw, std::size_t logical_len,
|
||||
if (is_bigram) {
|
||||
unit_width = 2;
|
||||
}
|
||||
const std::size_t raw_count =
|
||||
is_bigram && logical_len > 0 ? logical_len + 1 : logical_len * unit_width;
|
||||
if (has_wide_tokens(raw, raw_count)) {
|
||||
hash_wide_page(raw, is_bigram ? logical_len : raw_count, is_bigram,
|
||||
has_prior_digest, prior_digest);
|
||||
return digest_to_hex_string(prior_digest.data());
|
||||
}
|
||||
const bool can_hash_raw_bytes =
|
||||
std::is_same_v<RawToken, std::uint32_t> && !is_bigram;
|
||||
const unsigned char *bytes = nullptr;
|
||||
|
||||
@@ -60,7 +60,7 @@ def _native_hash_input(token_ids: Any) -> tuple[array, int, int, bool]:
|
||||
return raw, logical_len, 2 if is_bigram else 1, is_bigram
|
||||
|
||||
if is_bigram:
|
||||
return array("I", raw[: logical_len + 1]), logical_len, 2, is_bigram
|
||||
return array("q", raw[: logical_len + 1]), logical_len, 2, is_bigram
|
||||
|
||||
if logical_len == 0:
|
||||
return array("I"), logical_len, 1, is_bigram
|
||||
@@ -69,13 +69,13 @@ def _native_hash_input(token_ids: Any) -> tuple[array, int, int, bool]:
|
||||
if isinstance(first_token, tuple):
|
||||
unit_width = len(first_token)
|
||||
return (
|
||||
array("I", (elem for token in raw[:logical_len] for elem in token)),
|
||||
array("q", (elem for token in raw[:logical_len] for elem in token)),
|
||||
logical_len,
|
||||
unit_width,
|
||||
is_bigram,
|
||||
)
|
||||
|
||||
return array("I", raw[:logical_len]), logical_len, 1, is_bigram
|
||||
return array("q", raw[:logical_len]), logical_len, 1, is_bigram
|
||||
|
||||
|
||||
def get_native_hash(
|
||||
|
||||
@@ -652,12 +652,12 @@ class Ernie4_5_VLMoeForConditionalGeneration(nn.Module):
|
||||
)
|
||||
|
||||
pad_values = []
|
||||
if hasattr(forward_batch, "mm_inputs") and forward_batch.mm_inputs is not None:
|
||||
if forward_batch.mm_inputs is not None:
|
||||
for mm_input in forward_batch.mm_inputs:
|
||||
if mm_input is None:
|
||||
continue
|
||||
for item in mm_input.mm_items:
|
||||
pad_values.append(item.pad_value)
|
||||
pad_values.extend(item.padding_values())
|
||||
placeholder_tensor = torch.as_tensor(
|
||||
pad_values,
|
||||
device=input_ids.device,
|
||||
|
||||
@@ -3,16 +3,72 @@
|
||||
Shared by KimiVLImageProcessor and KimiK2_5VLImageProcessor.
|
||||
"""
|
||||
|
||||
from typing import Optional, Union
|
||||
import hashlib
|
||||
from collections.abc import Mapping
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, List, Optional, Sequence, Union
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.managers.io_struct import GenerateReqInput
|
||||
from sglang.srt.managers.schedule_batch import (
|
||||
Modality,
|
||||
MultimodalDataItem,
|
||||
MultimodalInputFormat,
|
||||
MultimodalProcessorOutput,
|
||||
)
|
||||
from sglang.srt.multimodal.cache import (
|
||||
CONTENT_HASH_PREFIX,
|
||||
build_artifact_key,
|
||||
build_processor_fingerprint,
|
||||
parse_content_hash,
|
||||
snapshot_media,
|
||||
)
|
||||
|
||||
_KIMI_IMAGE_IDENTITY_TAG = b"sglang-kimi-image-identity-v1"
|
||||
_CALLER_IDENTITY_KEYS = ("hash", "pad_value", "pad_values", "identity")
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class KimiLoadedImage:
|
||||
"""Decoded image and the digest of the exact input bytes read by its loader."""
|
||||
|
||||
image: Any
|
||||
content_digest: str
|
||||
|
||||
|
||||
def unwrap_kimi_images(images):
|
||||
return [
|
||||
image.image if isinstance(image, KimiLoadedImage) else image for image in images
|
||||
]
|
||||
|
||||
|
||||
def kimi_image_identity(content_config_digest: str, grid_thw: Sequence[int]) -> bytes:
|
||||
"""Full SHA-256 image identity: content and preprocessing config, then grid."""
|
||||
digest = parse_content_hash(content_config_digest)
|
||||
if digest is None:
|
||||
raise ValueError("Kimi image identity requires a content digest")
|
||||
hasher = hashlib.sha256(_KIMI_IMAGE_IDENTITY_TAG)
|
||||
hasher.update(bytes.fromhex(digest[len(CONTENT_HASH_PREFIX) :]))
|
||||
for dim in grid_thw:
|
||||
hasher.update(int(dim).to_bytes(8, byteorder="big", signed=True))
|
||||
return hasher.digest()
|
||||
|
||||
|
||||
def _item_grid_thw(item: MultimodalDataItem) -> List[int]:
|
||||
grid = item.model_specific_data.get("image_grid_thw")
|
||||
if grid is None:
|
||||
grid = item.model_specific_data.get("grid_thws")
|
||||
if grid is None:
|
||||
raise ValueError("Kimi image item is missing image_grid_thw")
|
||||
values = torch.as_tensor(grid).reshape(-1).tolist()
|
||||
if len(values) == 2:
|
||||
values.insert(0, 1)
|
||||
if len(values) != 3:
|
||||
raise ValueError(f"Kimi image item needs one [t, h, w] grid, got {values}")
|
||||
return values
|
||||
|
||||
|
||||
class KimiGridMMDataMixin:
|
||||
@@ -23,6 +79,102 @@ class KimiGridMMDataMixin:
|
||||
- self._tokenizer (with .encode())
|
||||
"""
|
||||
|
||||
# Opt-in: processors whose image spans use full identities and wide pads.
|
||||
uses_wide_image_identity = False
|
||||
|
||||
@classmethod
|
||||
def _load_single_item(cls, data, modality, *args, **kwargs):
|
||||
if (
|
||||
not cls.uses_wide_image_identity
|
||||
or modality != Modality.IMAGE
|
||||
or cls._is_preprocessed_input(data)
|
||||
or envs.SGLANG_MM_SKIP_COMPUTE_HASH.get()
|
||||
):
|
||||
return super()._load_single_item(data, modality, *args, **kwargs)
|
||||
# hash the bytes already on the host, before nvJPEG creates a GPU tensor
|
||||
snapshot = snapshot_media(data)
|
||||
image = super()._load_single_item(snapshot.data, modality, *args, **kwargs)
|
||||
return KimiLoadedImage(image, snapshot.content_digest)
|
||||
|
||||
def process_mm_data(self, input_text, images=None, **kwargs):
|
||||
return super().process_mm_data(
|
||||
input_text,
|
||||
images=unwrap_kimi_images(images) if images else images,
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def reject_caller_image_identity(image_data, request_obj) -> None:
|
||||
"""Image identities must come from this processor, never from callers."""
|
||||
if isinstance(request_obj, GenerateReqInput) and request_obj.mm_hashes:
|
||||
raise ValueError("Caller-supplied mm_hashes are not supported for Kimi.")
|
||||
for item in image_data or []:
|
||||
if isinstance(item, Mapping):
|
||||
for key in _CALLER_IDENTITY_KEYS:
|
||||
if key in item:
|
||||
raise ValueError(
|
||||
f"Caller-supplied multimodal {key} is not supported "
|
||||
"for Kimi."
|
||||
)
|
||||
|
||||
def _kimi_config_fingerprint(self) -> str:
|
||||
fingerprint = self.processor_fingerprint
|
||||
if fingerprint is None:
|
||||
fingerprint = build_processor_fingerprint(self, self.hf_config)
|
||||
self.processor_fingerprint = fingerprint
|
||||
return fingerprint
|
||||
|
||||
def kimi_content_config_digest(self, media: Any) -> str:
|
||||
return build_artifact_key(
|
||||
media.content_digest
|
||||
if isinstance(media, KimiLoadedImage)
|
||||
else snapshot_media(media).content_digest,
|
||||
modality="image",
|
||||
processor_fingerprint=self._kimi_config_fingerprint(),
|
||||
)
|
||||
|
||||
def assign_kimi_image_identities(
|
||||
self, mm_items: List[MultimodalDataItem], images: Optional[List[Any]]
|
||||
) -> None:
|
||||
"""Give each image item a full content+config+grid identity.
|
||||
|
||||
Source images identify processor-computed items; preprocessed or
|
||||
precomputed inputs are identified by their own tensor contents.
|
||||
"""
|
||||
if envs.SGLANG_MM_SKIP_COMPUTE_HASH.get():
|
||||
return
|
||||
image_items = [item for item in mm_items if item.is_image()]
|
||||
use_sources = (
|
||||
images is not None
|
||||
and len(images) == len(image_items)
|
||||
and all(item.format == MultimodalInputFormat.NORMAL for item in image_items)
|
||||
)
|
||||
for index, item in enumerate(image_items):
|
||||
if use_sources:
|
||||
source = images[index]
|
||||
elif item.feature is not None:
|
||||
source = item.feature
|
||||
else:
|
||||
source = item.precomputed_embeddings
|
||||
item.set_identity(
|
||||
kimi_image_identity(
|
||||
self.kimi_content_config_digest(source), _item_grid_thw(item)
|
||||
)
|
||||
)
|
||||
|
||||
def _postprocess_mm_items_before_transport(
|
||||
self,
|
||||
mm_items: List[MultimodalDataItem],
|
||||
*,
|
||||
images: Optional[List[Any]],
|
||||
) -> List[MultimodalDataItem]:
|
||||
mm_items = super()._postprocess_mm_items_before_transport(
|
||||
mm_items, images=images
|
||||
)
|
||||
if self.uses_wide_image_identity:
|
||||
self.assign_kimi_image_identities(mm_items, images)
|
||||
return mm_items
|
||||
|
||||
def resolve_image_token_counts(self, images):
|
||||
"""Kimi's processor is remote-code and does not implement the
|
||||
transformers ``_get_num_multimodal_tokens`` convention; use its
|
||||
@@ -30,6 +182,7 @@ class KimiGridMMDataMixin:
|
||||
|
||||
"""
|
||||
assert images is not None
|
||||
images = unwrap_kimi_images(images)
|
||||
media_tokens_calculator = (
|
||||
self._processor.media_processor.media_tokens_calculator
|
||||
)
|
||||
@@ -125,7 +278,7 @@ class KimiGridMMDataMixin:
|
||||
image_embeddings = embeddings[Modality.IMAGE]
|
||||
mm_items = []
|
||||
consumed = 0
|
||||
for start, end in offsets:
|
||||
for (start, end), grid in zip(offsets, img_grid_thw):
|
||||
num_tokens = end - start + 1
|
||||
embedding_slice = image_embeddings[consumed : consumed + num_tokens]
|
||||
consumed += num_tokens
|
||||
@@ -134,9 +287,13 @@ class KimiGridMMDataMixin:
|
||||
modality=Modality.IMAGE,
|
||||
offsets=[(start, end)],
|
||||
precomputed_embeddings=embedding_slice,
|
||||
model_specific_data={"image_grid_thw": grid},
|
||||
)
|
||||
)
|
||||
|
||||
if self.uses_wide_image_identity:
|
||||
self.assign_kimi_image_identities(mm_items, None)
|
||||
|
||||
return MultimodalProcessorOutput(
|
||||
input_ids=input_ids,
|
||||
mm_items=mm_items,
|
||||
|
||||
@@ -10,9 +10,7 @@ from PIL import Image
|
||||
|
||||
from sglang.kernels.ops.mm.process import normalize_and_patchify
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.managers.schedule_batch import (
|
||||
MultimodalProcessorOutput,
|
||||
)
|
||||
from sglang.srt.managers.schedule_batch import MultimodalProcessorOutput
|
||||
from sglang.srt.models.kimi_k25 import KimiK25ForConditionalGeneration
|
||||
from sglang.srt.multimodal.processors.base_processor import (
|
||||
BaseMultimodalProcessor as SGLangBaseProcessor,
|
||||
@@ -387,6 +385,20 @@ class KimiGPUProcessorWrapper:
|
||||
self.tokenizer = hf_processor.tokenizer
|
||||
self.media_processor = hf_processor.media_processor
|
||||
|
||||
def preprocess_fingerprint_payload(self):
|
||||
return {
|
||||
"patch_size": self._patch_size,
|
||||
"merge_kernel_size": self._merge_kernel_size,
|
||||
"in_patch_limit": self._in_patch_limit,
|
||||
"patch_limit_on_one_side": self._patch_limit_on_one_side,
|
||||
"fixed_output_tokens": self._fixed_output_tokens,
|
||||
"image_mean": self._image_mean,
|
||||
"image_std": self._image_std,
|
||||
"hf_media_config": self.media_processor.media_proc_cfg,
|
||||
"force_cpu": _FORCE_CPU_IMAGE_PREPROCESSING,
|
||||
"cuda_available": torch.cuda.is_available(),
|
||||
}
|
||||
|
||||
def __call__(self, text=None, images=None, **kwargs):
|
||||
# process_mm_data passes images via kwargs["images"]
|
||||
images = images or kwargs.pop("images", None)
|
||||
@@ -521,6 +533,7 @@ class KimiK2_5VLImageProcessor(KimiGridMMDataMixin, SGLangBaseProcessor):
|
||||
auto_mm_processor_worker_num = 2
|
||||
auto_mm_io_worker_num = 16
|
||||
supports_mm_processor_concurrency = True
|
||||
uses_wide_image_identity = True
|
||||
|
||||
def __init__(self, hf_config, server_args, _processor, *args, **kwargs):
|
||||
mm_tokens = MultimodalSpecialTokens(
|
||||
@@ -556,6 +569,7 @@ class KimiK2_5VLImageProcessor(KimiGridMMDataMixin, SGLangBaseProcessor):
|
||||
*args,
|
||||
**kwargs,
|
||||
):
|
||||
self.reject_caller_image_identity(image_data, request_obj)
|
||||
expected_image_count = len(image_data or [])
|
||||
placeholder_count = self.count_image_placeholders(
|
||||
input_text, self.mm_tokens.image_token_id
|
||||
|
||||
@@ -17,13 +17,14 @@ import numpy as np
|
||||
import torch
|
||||
from PIL import Image
|
||||
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.managers.schedule_batch import (
|
||||
Modality,
|
||||
MultimodalDataItem,
|
||||
MultimodalProcessorOutput,
|
||||
)
|
||||
from sglang.srt.models.kimi_k3 import KimiK3ForConditionalGeneration
|
||||
from sglang.srt.multimodal.cache import resolve_multimodal_item_hash
|
||||
from sglang.srt.multimodal.cache import resolve_multimodal_item_hash, snapshot_media
|
||||
from sglang.srt.multimodal.kimi_k3_image_processing import (
|
||||
DEFERRED_PREPROCESSING_KEY,
|
||||
KimiK3DeferredPreprocessing,
|
||||
@@ -49,7 +50,12 @@ from sglang.srt.multimodal.processors.base_processor import (
|
||||
from sglang.srt.multimodal.processors.base_processor import (
|
||||
MultimodalSpecialTokens,
|
||||
)
|
||||
from sglang.srt.multimodal.processors.kimi_common import KimiGridMMDataMixin
|
||||
from sglang.srt.multimodal.processors.kimi_common import (
|
||||
KimiGridMMDataMixin,
|
||||
KimiLoadedImage,
|
||||
kimi_image_identity,
|
||||
unwrap_kimi_images,
|
||||
)
|
||||
from sglang.srt.multimodal.processors.kimi_k25 import (
|
||||
KimiGPUProcessorWrapper,
|
||||
_get_image_dimensions,
|
||||
@@ -381,6 +387,7 @@ class KimiK3ImageProcessor(
|
||||
auto_mm_preprocess_cache_size_mb = 256
|
||||
supports_mm_processor_concurrency = True
|
||||
preserve_processor_input_ids = True
|
||||
uses_wide_image_identity = True
|
||||
|
||||
def __init__(self, hf_config, server_args, _processor, *args, **kwargs):
|
||||
mm_tokens = MultimodalSpecialTokens(
|
||||
@@ -406,6 +413,7 @@ class KimiK3ImageProcessor(
|
||||
"""
|
||||
when raw_bytes <= processed_bytes, preprocess first would introduce larger payload, so deferring gpu preprocessing would benefit
|
||||
"""
|
||||
images = unwrap_kimi_images(images or [])
|
||||
if (
|
||||
not images
|
||||
or self.mm_feature_transport != "cpu"
|
||||
@@ -452,13 +460,14 @@ class KimiK3ImageProcessor(
|
||||
return raw_bytes <= processed_bytes
|
||||
|
||||
def _build_deferred_output(self, base_output):
|
||||
images = unwrap_kimi_images(base_output.images)
|
||||
(
|
||||
input_ids,
|
||||
resize_configs,
|
||||
deferred_preprocessing,
|
||||
) = self._processor.prepare_deferred(
|
||||
base_output.input_text,
|
||||
base_output.images,
|
||||
images,
|
||||
base_output.input_ids,
|
||||
)
|
||||
offsets = self.get_mm_items_offset(
|
||||
@@ -468,9 +477,7 @@ class KimiK3ImageProcessor(
|
||||
raise ValueError("Expected one Kimi-K3 image span for each image")
|
||||
|
||||
items = []
|
||||
for image, resize_config, offset in zip(
|
||||
base_output.images, resize_configs, offsets
|
||||
):
|
||||
for image, resize_config, offset in zip(images, resize_configs, offsets):
|
||||
grid_thw = _grid_thw_from_resize_config(
|
||||
resize_config, self._processor.preprocess_config.patch_size
|
||||
)
|
||||
@@ -487,6 +494,7 @@ class KimiK3ImageProcessor(
|
||||
)
|
||||
items.append(item)
|
||||
|
||||
self.assign_kimi_image_identities(items, base_output.images)
|
||||
self._precompute_hashes_before_cpu_transfer(items)
|
||||
return MultimodalProcessorOutput(
|
||||
input_ids=input_ids.flatten().tolist(),
|
||||
@@ -644,7 +652,12 @@ class KimiK3ImageProcessor(
|
||||
offsets=[offset],
|
||||
model_specific_data=model_specific_data,
|
||||
)
|
||||
item.set_hash(artifact.feature_hash)
|
||||
if envs.SGLANG_MM_SKIP_COMPUTE_HASH.get():
|
||||
item.set_hash(artifact.feature_hash)
|
||||
else:
|
||||
item.set_identity(
|
||||
kimi_image_identity(artifact.artifact_key, artifact.grid_thw)
|
||||
)
|
||||
if self.keep_mm_features_on_device and item.feature is not None:
|
||||
item.model_specific_data[DEFER_CUDA_IPC_FEATURE_RECONSTRUCTION_KEY] = (
|
||||
True
|
||||
@@ -703,6 +716,17 @@ class KimiK3ImageProcessor(
|
||||
im_token_id=self.mm_tokens.image_token_id,
|
||||
)
|
||||
|
||||
def kimi_content_config_digest(self, media) -> str:
|
||||
self._kimi_config_fingerprint()
|
||||
digest = (
|
||||
media.content_digest
|
||||
if isinstance(media, KimiLoadedImage)
|
||||
else snapshot_media(media).content_digest
|
||||
)
|
||||
return self._artifact_key(
|
||||
digest, media.image if isinstance(media, KimiLoadedImage) else media
|
||||
)
|
||||
|
||||
async def process_mm_data_async(
|
||||
self,
|
||||
image_data: List[Union[str, bytes, Dict]],
|
||||
@@ -713,6 +737,7 @@ class KimiK3ImageProcessor(
|
||||
):
|
||||
if request_obj.video_data or kwargs.get("audio_data"):
|
||||
raise ValueError("Kimi-K3 supports image input only")
|
||||
self.reject_caller_image_identity(image_data, request_obj)
|
||||
|
||||
expected_image_count = len(image_data or [])
|
||||
placeholder_count = self.count_image_placeholders(
|
||||
|
||||
@@ -252,14 +252,17 @@ class NGRAMWorker(BaseSpecWorker):
|
||||
req_ids = [req.rid for req in batch.reqs]
|
||||
# Only the last max_trie_depth tokens can match; the ids are
|
||||
# array.array, so list() the tails for list concat below.
|
||||
# multimodal cache padding contains identities, not vocabulary token IDs
|
||||
input_tails = [
|
||||
list(req.origin_input_ids[-self.max_trie_depth :]) for req in batch.reqs
|
||||
list(req.origin_input_ids_unpadded[-self.max_trie_depth :])
|
||||
for req in batch.reqs
|
||||
]
|
||||
output_tails = [
|
||||
list(req.output_ids[-self.max_trie_depth :]) for req in batch.reqs
|
||||
]
|
||||
base_lens = [
|
||||
len(req.origin_input_ids) + len(req.output_ids) for req in batch.reqs
|
||||
len(req.origin_input_ids_unpadded) + len(req.output_ids)
|
||||
for req in batch.reqs
|
||||
]
|
||||
|
||||
if use_prev_tokens:
|
||||
@@ -414,7 +417,7 @@ class NGRAMWorker(BaseSpecWorker):
|
||||
else []
|
||||
)
|
||||
put_ids = self._efficient_concat_last_n(
|
||||
list(req.origin_input_ids[-self.max_trie_depth :]),
|
||||
list(req.origin_input_ids_unpadded[-self.max_trie_depth :]),
|
||||
list(req.output_ids[-self.max_trie_depth :]) + prev_tokens,
|
||||
self.max_trie_depth,
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user