[Fix] VLM: harden Kimi image cache identity across preprocessing and radix hashing (#43043)

Co-authored-by: Rahul Chalamala <22563365+rchalamala@users.noreply.github.com>
Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
Co-authored-by: Mick Qian <mickqian@users.noreply.github.com>
Co-authored-by: Liangsheng Yin <hnyls2002@gmail.com>
This commit is contained in:
Mick
2026-10-18 17:39:37 +08:00
committed by GitHub
co-authored by Rahul Chalamala Devin AI Mick Qian Liangsheng Yin
parent 5a930cfd88
commit 3b932cde26
21 changed files with 1115 additions and 89 deletions
+11 -7
View File
@@ -266,7 +266,7 @@ def pad_mm_token_pairs(
if data_start_token_ids is None:
data_start_token_ids = [s for s, _e in data_token_pairs]
assert_int64_array(input_ids, "input_ids")
pad_values = [item.pad_value for item in mm_inputs.mm_items]
pad_items = mm_inputs.mm_items
mm_inputs.data_offsets = []
if data_token_pairs is None:
data_token_pairs = [mm_inputs.im_start_id, mm_inputs.im_end_id]
@@ -295,13 +295,12 @@ def pad_mm_token_pairs(
data_idx += 1
mm_inputs.data_offsets += [start_idx]
if data_idx >= len(pad_values):
data_idx = len(pad_values) - 1
if data_idx >= len(pad_items):
data_idx = len(pad_items) - 1
num_tokens = end_idx - start_idx - 1
pad_value = pad_values[data_idx]
if num_tokens > 0:
padded_ids.extend(array("q", [pad_value]) * num_tokens)
padded_ids.extend(pad_items[data_idx].padding_sequence(num_tokens))
last_idx = end_idx
@@ -341,7 +340,10 @@ def pad_mm_tokens(input_ids: array, mm_inputs: MultimodalInputs) -> array:
for i, item in enumerate(items):
for offset in items[i].offsets:
input_ids_tensor[offset[0] : offset[1] + 1] = item.pad_value
input_ids_tensor[offset[0] : offset[1] + 1] = torch.as_tensor(
item.padding_sequence(offset[1] - offset[0] + 1),
dtype=input_ids_tensor.dtype,
)
return padded_input_ids
@@ -493,7 +495,7 @@ def embed_mm_inputs(
embedder = getattr(multimodal_model, f"get_{modality_id}_feature", None)
if len(items) != 0:
assert embedder is not None, f"no embedding method found for {modality}"
pad_values = [item.pad_value for item in items]
pad_values = [value for item in items for value in item.padding_values()]
if input_ids.device.type == "cuda":
# Pinned staging keeps the placeholder copy asynchronous on CUDA.
placeholder_cpu = torch.tensor(
@@ -1189,6 +1191,8 @@ def get_new_expanded_mm_items(original_mm_items):
# explicit None check, not `a or b`: the value is a multi-element
# tensor whose truthiness is ambiguous.)
image_grid_thw = item.model_specific_data.get("image_grid_thw")
if image_grid_thw is None:
image_grid_thw = item.model_specific_data.get("grid_thws")
if image_grid_thw is None:
image_grid_thw = item.model_specific_data.get("image_grid_hws")
grid_len = _get_length(image_grid_thw)
+55 -10
View File
@@ -53,6 +53,7 @@ ScheduleBatch -> ForwardBatch
import copy
import dataclasses
import hashlib
import logging
import re
import sys
@@ -160,6 +161,10 @@ INIT_INCREMENTAL_DETOKENIZATION_OFFSET = 5
# This ensures pad_values don't overlap with valid text token IDs.
MM_PAD_SHIFT_VALUE = 1_000_000
_MM_HASH_MASK = (1 << 64) - 1
_MM_IDENTITY_BYTES = 32
_MM_WIDE_PAD_TAG = b"sglang-mm-wide-pad-v1"
# Keep MM_PAD_SHIFT_VALUE + sentinel below 2**63 so it fits an int64 token ID.
_MM_WIDE_PAD_MASK = (1 << 62) - 1
logger = logging.getLogger(__name__)
@@ -255,9 +260,15 @@ def split_cached_prefix_by_tier(
return device, host, storage
def _compute_pad_value(hash: int) -> int:
"""Compute pad value from hash."""
return MM_PAD_SHIFT_VALUE + (hash % (1 << 30))
def _compute_pad_values(identity: bytes) -> Tuple[int, ...]:
"""derive shared wide sentinels from an item hash or full identity digest"""
digest = hashlib.sha256(_MM_WIDE_PAD_TAG + identity).digest()
# prefix matching can stop after one sentinel, so each must retain wide bits
return tuple(
MM_PAD_SHIFT_VALUE
+ (int.from_bytes(digest[i : i + 8], byteorder="big") & _MM_WIDE_PAD_MASK)
for i in range(0, _MM_IDENTITY_BYTES, 8)
)
class BaseFinishReason:
@@ -391,6 +402,11 @@ class MultimodalDataItem(msgspec.Struct, kw_only=True, dict=True, array_like=Tru
default_factory=dict
)
# optional full identity; otherwise sentinels derive from the 64-bit item hash
# pad_value remains the first sentinel for scalar-padding consumers
identity: Optional[bytes] = None
pad_values: Optional[Tuple[int, ...]] = None
def __post_init__(self) -> None:
if self.hash is not None:
msgspec.Struct.__setattr__(self, "hash", self.hash & _MM_HASH_MASK)
@@ -419,7 +435,33 @@ class MultimodalDataItem(msgspec.Struct, kw_only=True, dict=True, array_like=Tru
def set_hash(self, hash_value: int) -> None:
self.hash = hash_value
self.pad_value = _compute_pad_value(hash_value)
self.identity = None
self.pad_values = _compute_pad_values(self.hash.to_bytes(8, byteorder="big"))
self.pad_value = self.pad_values[0]
def set_identity(self, identity: bytes) -> None:
"""Adopt a full 32-byte digest and derive four 62-bit pad sentinels.
``hash`` keeps the leading 64 bits for scalar consumers such as the
embedding cache; the radix key uses the full sentinel pattern.
"""
if len(identity) != _MM_IDENTITY_BYTES:
raise ValueError(
f"multimodal identity must be {_MM_IDENTITY_BYTES} bytes, "
f"got {len(identity)}"
)
self.identity = bytes(identity)
self.hash = int.from_bytes(identity[:8], byteorder="big", signed=False)
self.pad_values = _compute_pad_values(self.identity)
self.pad_value = self.pad_values[0]
def padding_values(self) -> Tuple[int, ...]:
return self.pad_values or (self.pad_value,)
def padding_sequence(self, length: int) -> List[int]:
values = self.padding_values()
repeats, remainder = divmod(length, len(values))
return list(values) * repeats + list(values[:remainder])
@staticmethod
def is_empty_list(l):
@@ -433,12 +475,13 @@ class MultimodalDataItem(msgspec.Struct, kw_only=True, dict=True, array_like=Tru
from sglang.srt.multimodal.cache import resolve_multimodal_item_hash
self.hash = resolve_multimodal_item_hash(
existing_hash=self.hash,
feature=self.feature,
precomputed_embeddings=self.precomputed_embeddings,
self.set_hash(
resolve_multimodal_item_hash(
existing_hash=self.hash,
feature=self.feature,
precomputed_embeddings=self.precomputed_embeddings,
)
)
self.pad_value = _compute_pad_value(self.hash)
def is_modality(self, modality: Modality) -> bool:
return self.modality == modality
@@ -650,7 +693,9 @@ class MultimodalProcessorOutput(
for item in mm_items:
for start, end in item.offsets:
padded_input_ids[start : end + 1] = [item.pad_value] * (end - start + 1)
padded_input_ids[start : end + 1] = item.padding_sequence(
end - start + 1
)
return padded_input_ids
@staticmethod
@@ -61,12 +61,18 @@ parse_prior_digest(py::object prior_digest_obj, bool *has_prior_digest) {
inline void hash_page(const unsigned char *data, std::size_t len,
bool &has_prior_digest,
std::array<unsigned char, kDigestLen> &prior_digest) {
std::array<unsigned char, kDigestLen> &prior_digest,
bool wide = false) {
SHA256_CTX ctx;
SHA256_Init(&ctx);
if (has_prior_digest) {
SHA256_Update(&ctx, prior_digest.data(), prior_digest.size());
}
if (wide) {
// an odd byte count separates wide encoding from legacy u32 pages
const unsigned char tag = 0xff;
SHA256_Update(&ctx, &tag, sizeof(tag));
}
if (len > 0) {
SHA256_Update(&ctx, data, len);
}
@@ -74,6 +80,40 @@ inline void hash_page(const unsigned char *data, std::size_t len,
has_prior_digest = true;
}
template <typename RawToken>
bool has_wide_tokens(const RawToken *raw, std::size_t count) {
if constexpr (sizeof(RawToken) == 4) {
return false;
}
bool wide = false;
for (std::size_t i = 0; i < count; ++i) {
if (raw[i] > INT64_MAX) {
throw std::out_of_range("token id must be a non-negative int64");
}
wide |= raw[i] > UINT32_MAX;
}
return wide;
}
template <typename RawToken>
void hash_wide_page(const RawToken *raw, std::size_t count, bool is_bigram,
bool &has_prior_digest,
std::array<unsigned char, kDigestLen> &prior_digest) {
std::vector<std::uint64_t> words;
if (is_bigram) {
words.reserve(count * 2);
for (std::size_t i = 0; i < count; ++i) {
words.push_back(raw[i]);
words.push_back(raw[i + 1]);
}
} else {
words.assign(raw, raw + count);
}
hash_page(reinterpret_cast<const unsigned char *>(words.data()),
words.size() * sizeof(std::uint64_t), has_prior_digest,
prior_digest, true);
}
template <typename RawToken>
inline void fill_regular_page(const RawToken *raw, std::size_t start,
std::size_t count, std::uint32_t *out) {
@@ -203,6 +243,16 @@ void hash_pages_to_hex_blob(const RawToken *raw, std::size_t logical_len,
for (std::size_t start = 0, page_idx = 0; start < logical_len;
start += page_size, ++page_idx) {
const std::size_t page_units = std::min(page_size, logical_len - start);
const auto *page_start = raw + (is_bigram ? start : start * unit_width);
const std::size_t raw_count =
is_bigram ? page_units + 1 : page_units * unit_width;
if (has_wide_tokens(page_start, raw_count)) {
hash_wide_page(page_start, is_bigram ? page_units : raw_count, is_bigram,
has_prior_digest, prior_digest);
digest_to_hex_chars(prior_digest.data(),
hex_blob.data() + page_idx * kHexLen);
continue;
}
const std::size_t page_bytes =
page_units * unit_width * sizeof(std::uint32_t);
const unsigned char *bytes = nullptr;
@@ -233,6 +283,13 @@ std::string hash_all(const RawToken *raw, std::size_t logical_len,
if (is_bigram) {
unit_width = 2;
}
const std::size_t raw_count =
is_bigram && logical_len > 0 ? logical_len + 1 : logical_len * unit_width;
if (has_wide_tokens(raw, raw_count)) {
hash_wide_page(raw, is_bigram ? logical_len : raw_count, is_bigram,
has_prior_digest, prior_digest);
return digest_to_hex_string(prior_digest.data());
}
const bool can_hash_raw_bytes =
std::is_same_v<RawToken, std::uint32_t> && !is_bigram;
const unsigned char *bytes = nullptr;
@@ -60,7 +60,7 @@ def _native_hash_input(token_ids: Any) -> tuple[array, int, int, bool]:
return raw, logical_len, 2 if is_bigram else 1, is_bigram
if is_bigram:
return array("I", raw[: logical_len + 1]), logical_len, 2, is_bigram
return array("q", raw[: logical_len + 1]), logical_len, 2, is_bigram
if logical_len == 0:
return array("I"), logical_len, 1, is_bigram
@@ -69,13 +69,13 @@ def _native_hash_input(token_ids: Any) -> tuple[array, int, int, bool]:
if isinstance(first_token, tuple):
unit_width = len(first_token)
return (
array("I", (elem for token in raw[:logical_len] for elem in token)),
array("q", (elem for token in raw[:logical_len] for elem in token)),
logical_len,
unit_width,
is_bigram,
)
return array("I", raw[:logical_len]), logical_len, 1, is_bigram
return array("q", raw[:logical_len]), logical_len, 1, is_bigram
def get_native_hash(
+2 -2
View File
@@ -652,12 +652,12 @@ class Ernie4_5_VLMoeForConditionalGeneration(nn.Module):
)
pad_values = []
if hasattr(forward_batch, "mm_inputs") and forward_batch.mm_inputs is not None:
if forward_batch.mm_inputs is not None:
for mm_input in forward_batch.mm_inputs:
if mm_input is None:
continue
for item in mm_input.mm_items:
pad_values.append(item.pad_value)
pad_values.extend(item.padding_values())
placeholder_tensor = torch.as_tensor(
pad_values,
device=input_ids.device,
@@ -3,16 +3,72 @@
Shared by KimiVLImageProcessor and KimiK2_5VLImageProcessor.
"""
from typing import Optional, Union
import hashlib
from collections.abc import Mapping
from dataclasses import dataclass
from typing import Any, List, Optional, Sequence, Union
import numpy as np
import torch
from sglang.srt.environ import envs
from sglang.srt.managers.io_struct import GenerateReqInput
from sglang.srt.managers.schedule_batch import (
Modality,
MultimodalDataItem,
MultimodalInputFormat,
MultimodalProcessorOutput,
)
from sglang.srt.multimodal.cache import (
CONTENT_HASH_PREFIX,
build_artifact_key,
build_processor_fingerprint,
parse_content_hash,
snapshot_media,
)
_KIMI_IMAGE_IDENTITY_TAG = b"sglang-kimi-image-identity-v1"
_CALLER_IDENTITY_KEYS = ("hash", "pad_value", "pad_values", "identity")
@dataclass(frozen=True)
class KimiLoadedImage:
"""Decoded image and the digest of the exact input bytes read by its loader."""
image: Any
content_digest: str
def unwrap_kimi_images(images):
return [
image.image if isinstance(image, KimiLoadedImage) else image for image in images
]
def kimi_image_identity(content_config_digest: str, grid_thw: Sequence[int]) -> bytes:
"""Full SHA-256 image identity: content and preprocessing config, then grid."""
digest = parse_content_hash(content_config_digest)
if digest is None:
raise ValueError("Kimi image identity requires a content digest")
hasher = hashlib.sha256(_KIMI_IMAGE_IDENTITY_TAG)
hasher.update(bytes.fromhex(digest[len(CONTENT_HASH_PREFIX) :]))
for dim in grid_thw:
hasher.update(int(dim).to_bytes(8, byteorder="big", signed=True))
return hasher.digest()
def _item_grid_thw(item: MultimodalDataItem) -> List[int]:
grid = item.model_specific_data.get("image_grid_thw")
if grid is None:
grid = item.model_specific_data.get("grid_thws")
if grid is None:
raise ValueError("Kimi image item is missing image_grid_thw")
values = torch.as_tensor(grid).reshape(-1).tolist()
if len(values) == 2:
values.insert(0, 1)
if len(values) != 3:
raise ValueError(f"Kimi image item needs one [t, h, w] grid, got {values}")
return values
class KimiGridMMDataMixin:
@@ -23,6 +79,102 @@ class KimiGridMMDataMixin:
- self._tokenizer (with .encode())
"""
# Opt-in: processors whose image spans use full identities and wide pads.
uses_wide_image_identity = False
@classmethod
def _load_single_item(cls, data, modality, *args, **kwargs):
if (
not cls.uses_wide_image_identity
or modality != Modality.IMAGE
or cls._is_preprocessed_input(data)
or envs.SGLANG_MM_SKIP_COMPUTE_HASH.get()
):
return super()._load_single_item(data, modality, *args, **kwargs)
# hash the bytes already on the host, before nvJPEG creates a GPU tensor
snapshot = snapshot_media(data)
image = super()._load_single_item(snapshot.data, modality, *args, **kwargs)
return KimiLoadedImage(image, snapshot.content_digest)
def process_mm_data(self, input_text, images=None, **kwargs):
return super().process_mm_data(
input_text,
images=unwrap_kimi_images(images) if images else images,
**kwargs,
)
@staticmethod
def reject_caller_image_identity(image_data, request_obj) -> None:
"""Image identities must come from this processor, never from callers."""
if isinstance(request_obj, GenerateReqInput) and request_obj.mm_hashes:
raise ValueError("Caller-supplied mm_hashes are not supported for Kimi.")
for item in image_data or []:
if isinstance(item, Mapping):
for key in _CALLER_IDENTITY_KEYS:
if key in item:
raise ValueError(
f"Caller-supplied multimodal {key} is not supported "
"for Kimi."
)
def _kimi_config_fingerprint(self) -> str:
fingerprint = self.processor_fingerprint
if fingerprint is None:
fingerprint = build_processor_fingerprint(self, self.hf_config)
self.processor_fingerprint = fingerprint
return fingerprint
def kimi_content_config_digest(self, media: Any) -> str:
return build_artifact_key(
media.content_digest
if isinstance(media, KimiLoadedImage)
else snapshot_media(media).content_digest,
modality="image",
processor_fingerprint=self._kimi_config_fingerprint(),
)
def assign_kimi_image_identities(
self, mm_items: List[MultimodalDataItem], images: Optional[List[Any]]
) -> None:
"""Give each image item a full content+config+grid identity.
Source images identify processor-computed items; preprocessed or
precomputed inputs are identified by their own tensor contents.
"""
if envs.SGLANG_MM_SKIP_COMPUTE_HASH.get():
return
image_items = [item for item in mm_items if item.is_image()]
use_sources = (
images is not None
and len(images) == len(image_items)
and all(item.format == MultimodalInputFormat.NORMAL for item in image_items)
)
for index, item in enumerate(image_items):
if use_sources:
source = images[index]
elif item.feature is not None:
source = item.feature
else:
source = item.precomputed_embeddings
item.set_identity(
kimi_image_identity(
self.kimi_content_config_digest(source), _item_grid_thw(item)
)
)
def _postprocess_mm_items_before_transport(
self,
mm_items: List[MultimodalDataItem],
*,
images: Optional[List[Any]],
) -> List[MultimodalDataItem]:
mm_items = super()._postprocess_mm_items_before_transport(
mm_items, images=images
)
if self.uses_wide_image_identity:
self.assign_kimi_image_identities(mm_items, images)
return mm_items
def resolve_image_token_counts(self, images):
"""Kimi's processor is remote-code and does not implement the
transformers ``_get_num_multimodal_tokens`` convention; use its
@@ -30,6 +182,7 @@ class KimiGridMMDataMixin:
"""
assert images is not None
images = unwrap_kimi_images(images)
media_tokens_calculator = (
self._processor.media_processor.media_tokens_calculator
)
@@ -125,7 +278,7 @@ class KimiGridMMDataMixin:
image_embeddings = embeddings[Modality.IMAGE]
mm_items = []
consumed = 0
for start, end in offsets:
for (start, end), grid in zip(offsets, img_grid_thw):
num_tokens = end - start + 1
embedding_slice = image_embeddings[consumed : consumed + num_tokens]
consumed += num_tokens
@@ -134,9 +287,13 @@ class KimiGridMMDataMixin:
modality=Modality.IMAGE,
offsets=[(start, end)],
precomputed_embeddings=embedding_slice,
model_specific_data={"image_grid_thw": grid},
)
)
if self.uses_wide_image_identity:
self.assign_kimi_image_identities(mm_items, None)
return MultimodalProcessorOutput(
input_ids=input_ids,
mm_items=mm_items,
@@ -10,9 +10,7 @@ from PIL import Image
from sglang.kernels.ops.mm.process import normalize_and_patchify
from sglang.srt.environ import envs
from sglang.srt.managers.schedule_batch import (
MultimodalProcessorOutput,
)
from sglang.srt.managers.schedule_batch import MultimodalProcessorOutput
from sglang.srt.models.kimi_k25 import KimiK25ForConditionalGeneration
from sglang.srt.multimodal.processors.base_processor import (
BaseMultimodalProcessor as SGLangBaseProcessor,
@@ -387,6 +385,20 @@ class KimiGPUProcessorWrapper:
self.tokenizer = hf_processor.tokenizer
self.media_processor = hf_processor.media_processor
def preprocess_fingerprint_payload(self):
return {
"patch_size": self._patch_size,
"merge_kernel_size": self._merge_kernel_size,
"in_patch_limit": self._in_patch_limit,
"patch_limit_on_one_side": self._patch_limit_on_one_side,
"fixed_output_tokens": self._fixed_output_tokens,
"image_mean": self._image_mean,
"image_std": self._image_std,
"hf_media_config": self.media_processor.media_proc_cfg,
"force_cpu": _FORCE_CPU_IMAGE_PREPROCESSING,
"cuda_available": torch.cuda.is_available(),
}
def __call__(self, text=None, images=None, **kwargs):
# process_mm_data passes images via kwargs["images"]
images = images or kwargs.pop("images", None)
@@ -521,6 +533,7 @@ class KimiK2_5VLImageProcessor(KimiGridMMDataMixin, SGLangBaseProcessor):
auto_mm_processor_worker_num = 2
auto_mm_io_worker_num = 16
supports_mm_processor_concurrency = True
uses_wide_image_identity = True
def __init__(self, hf_config, server_args, _processor, *args, **kwargs):
mm_tokens = MultimodalSpecialTokens(
@@ -556,6 +569,7 @@ class KimiK2_5VLImageProcessor(KimiGridMMDataMixin, SGLangBaseProcessor):
*args,
**kwargs,
):
self.reject_caller_image_identity(image_data, request_obj)
expected_image_count = len(image_data or [])
placeholder_count = self.count_image_placeholders(
input_text, self.mm_tokens.image_token_id
@@ -17,13 +17,14 @@ import numpy as np
import torch
from PIL import Image
from sglang.srt.environ import envs
from sglang.srt.managers.schedule_batch import (
Modality,
MultimodalDataItem,
MultimodalProcessorOutput,
)
from sglang.srt.models.kimi_k3 import KimiK3ForConditionalGeneration
from sglang.srt.multimodal.cache import resolve_multimodal_item_hash
from sglang.srt.multimodal.cache import resolve_multimodal_item_hash, snapshot_media
from sglang.srt.multimodal.kimi_k3_image_processing import (
DEFERRED_PREPROCESSING_KEY,
KimiK3DeferredPreprocessing,
@@ -49,7 +50,12 @@ from sglang.srt.multimodal.processors.base_processor import (
from sglang.srt.multimodal.processors.base_processor import (
MultimodalSpecialTokens,
)
from sglang.srt.multimodal.processors.kimi_common import KimiGridMMDataMixin
from sglang.srt.multimodal.processors.kimi_common import (
KimiGridMMDataMixin,
KimiLoadedImage,
kimi_image_identity,
unwrap_kimi_images,
)
from sglang.srt.multimodal.processors.kimi_k25 import (
KimiGPUProcessorWrapper,
_get_image_dimensions,
@@ -381,6 +387,7 @@ class KimiK3ImageProcessor(
auto_mm_preprocess_cache_size_mb = 256
supports_mm_processor_concurrency = True
preserve_processor_input_ids = True
uses_wide_image_identity = True
def __init__(self, hf_config, server_args, _processor, *args, **kwargs):
mm_tokens = MultimodalSpecialTokens(
@@ -406,6 +413,7 @@ class KimiK3ImageProcessor(
"""
when raw_bytes <= processed_bytes, preprocess first would introduce larger payload, so deferring gpu preprocessing would benefit
"""
images = unwrap_kimi_images(images or [])
if (
not images
or self.mm_feature_transport != "cpu"
@@ -452,13 +460,14 @@ class KimiK3ImageProcessor(
return raw_bytes <= processed_bytes
def _build_deferred_output(self, base_output):
images = unwrap_kimi_images(base_output.images)
(
input_ids,
resize_configs,
deferred_preprocessing,
) = self._processor.prepare_deferred(
base_output.input_text,
base_output.images,
images,
base_output.input_ids,
)
offsets = self.get_mm_items_offset(
@@ -468,9 +477,7 @@ class KimiK3ImageProcessor(
raise ValueError("Expected one Kimi-K3 image span for each image")
items = []
for image, resize_config, offset in zip(
base_output.images, resize_configs, offsets
):
for image, resize_config, offset in zip(images, resize_configs, offsets):
grid_thw = _grid_thw_from_resize_config(
resize_config, self._processor.preprocess_config.patch_size
)
@@ -487,6 +494,7 @@ class KimiK3ImageProcessor(
)
items.append(item)
self.assign_kimi_image_identities(items, base_output.images)
self._precompute_hashes_before_cpu_transfer(items)
return MultimodalProcessorOutput(
input_ids=input_ids.flatten().tolist(),
@@ -644,7 +652,12 @@ class KimiK3ImageProcessor(
offsets=[offset],
model_specific_data=model_specific_data,
)
item.set_hash(artifact.feature_hash)
if envs.SGLANG_MM_SKIP_COMPUTE_HASH.get():
item.set_hash(artifact.feature_hash)
else:
item.set_identity(
kimi_image_identity(artifact.artifact_key, artifact.grid_thw)
)
if self.keep_mm_features_on_device and item.feature is not None:
item.model_specific_data[DEFER_CUDA_IPC_FEATURE_RECONSTRUCTION_KEY] = (
True
@@ -703,6 +716,17 @@ class KimiK3ImageProcessor(
im_token_id=self.mm_tokens.image_token_id,
)
def kimi_content_config_digest(self, media) -> str:
self._kimi_config_fingerprint()
digest = (
media.content_digest
if isinstance(media, KimiLoadedImage)
else snapshot_media(media).content_digest
)
return self._artifact_key(
digest, media.image if isinstance(media, KimiLoadedImage) else media
)
async def process_mm_data_async(
self,
image_data: List[Union[str, bytes, Dict]],
@@ -713,6 +737,7 @@ class KimiK3ImageProcessor(
):
if request_obj.video_data or kwargs.get("audio_data"):
raise ValueError("Kimi-K3 supports image input only")
self.reject_caller_image_identity(image_data, request_obj)
expected_image_count = len(image_data or [])
placeholder_count = self.count_image_placeholders(
@@ -252,14 +252,17 @@ class NGRAMWorker(BaseSpecWorker):
req_ids = [req.rid for req in batch.reqs]
# Only the last max_trie_depth tokens can match; the ids are
# array.array, so list() the tails for list concat below.
# multimodal cache padding contains identities, not vocabulary token IDs
input_tails = [
list(req.origin_input_ids[-self.max_trie_depth :]) for req in batch.reqs
list(req.origin_input_ids_unpadded[-self.max_trie_depth :])
for req in batch.reqs
]
output_tails = [
list(req.output_ids[-self.max_trie_depth :]) for req in batch.reqs
]
base_lens = [
len(req.origin_input_ids) + len(req.output_ids) for req in batch.reqs
len(req.origin_input_ids_unpadded) + len(req.output_ids)
for req in batch.reqs
]
if use_prev_tokens:
@@ -414,7 +417,7 @@ class NGRAMWorker(BaseSpecWorker):
else []
)
put_ids = self._efficient_concat_last_n(
list(req.origin_input_ids[-self.max_trie_depth :]),
list(req.origin_input_ids_unpadded[-self.max_trie_depth :]),
list(req.output_ids[-self.max_trie_depth :]) + prev_tokens,
self.max_trie_depth,
)