From 3b932cde262945fb489e187f658fde7bbecd31a2 Mon Sep 17 00:00:00 2001 From: Mick Date: Sun, 18 Oct 2026 17:39:37 +0800 Subject: [PATCH] [Fix] VLM: harden Kimi image cache identity across preprocessing and radix hashing (#43043) Co-authored-by: Rahul Chalamala <22563365+rchalamala@users.noreply.github.com> Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Co-authored-by: Mick Qian Co-authored-by: Liangsheng Yin --- python/sglang/srt/managers/mm_utils.py | 18 +- python/sglang/srt/managers/schedule_batch.py | 65 +++- .../srt/mem_cache/cpp_utils/hash_binding.cpp | 59 +++- .../srt/mem_cache/cpp_utils/native_hash.py | 6 +- python/sglang/srt/models/ernie45_vl.py | 4 +- .../srt/multimodal/processors/kimi_common.py | 161 ++++++++- .../srt/multimodal/processors/kimi_k25.py | 20 +- .../srt/multimodal/processors/kimi_k3.py | 39 ++- python/sglang/srt/speculative/ngram_worker.py | 9 +- rust/sglang-radix-tree/src/node.rs | 30 +- rust/sglang-radix-tree/src/python_bindings.rs | 7 +- rust/sglang-radix-tree/src/tests/node.rs | 14 +- .../unit/managers/test_mm_hashes.py | 47 ++- .../unit/managers/test_mm_utils_split.py | 55 +++ .../unit/mem_cache/test_mem_cache_utils.py | 21 ++ .../test_rust_tree_core_integration.py | 69 +++- test/registered/unit/models/test_kimi_k25.py | 112 +++++- .../rust/shared/test_wrap_encoded.py | 12 +- .../multimodal/test_kimi_image_identity.py | 326 ++++++++++++++++++ .../unit/spec/test_ngram_worker_prompt.py | 87 +++++ test/registered/vlm/test_vlm_input_format.py | 43 +++ 21 files changed, 1115 insertions(+), 89 deletions(-) create mode 100644 test/registered/unit/multimodal/test_kimi_image_identity.py create mode 100644 test/registered/unit/spec/test_ngram_worker_prompt.py diff --git a/python/sglang/srt/managers/mm_utils.py b/python/sglang/srt/managers/mm_utils.py index f425223675..d4a8346afd 100644 --- a/python/sglang/srt/managers/mm_utils.py +++ b/python/sglang/srt/managers/mm_utils.py @@ -266,7 +266,7 @@ def pad_mm_token_pairs( if data_start_token_ids is None: data_start_token_ids = [s for s, _e in data_token_pairs] assert_int64_array(input_ids, "input_ids") - pad_values = [item.pad_value for item in mm_inputs.mm_items] + pad_items = mm_inputs.mm_items mm_inputs.data_offsets = [] if data_token_pairs is None: data_token_pairs = [mm_inputs.im_start_id, mm_inputs.im_end_id] @@ -295,13 +295,12 @@ def pad_mm_token_pairs( data_idx += 1 mm_inputs.data_offsets += [start_idx] - if data_idx >= len(pad_values): - data_idx = len(pad_values) - 1 + if data_idx >= len(pad_items): + data_idx = len(pad_items) - 1 num_tokens = end_idx - start_idx - 1 - pad_value = pad_values[data_idx] if num_tokens > 0: - padded_ids.extend(array("q", [pad_value]) * num_tokens) + padded_ids.extend(pad_items[data_idx].padding_sequence(num_tokens)) last_idx = end_idx @@ -341,7 +340,10 @@ def pad_mm_tokens(input_ids: array, mm_inputs: MultimodalInputs) -> array: for i, item in enumerate(items): for offset in items[i].offsets: - input_ids_tensor[offset[0] : offset[1] + 1] = item.pad_value + input_ids_tensor[offset[0] : offset[1] + 1] = torch.as_tensor( + item.padding_sequence(offset[1] - offset[0] + 1), + dtype=input_ids_tensor.dtype, + ) return padded_input_ids @@ -493,7 +495,7 @@ def embed_mm_inputs( embedder = getattr(multimodal_model, f"get_{modality_id}_feature", None) if len(items) != 0: assert embedder is not None, f"no embedding method found for {modality}" - pad_values = [item.pad_value for item in items] + pad_values = [value for item in items for value in item.padding_values()] if input_ids.device.type == "cuda": # Pinned staging keeps the placeholder copy asynchronous on CUDA. placeholder_cpu = torch.tensor( @@ -1189,6 +1191,8 @@ def get_new_expanded_mm_items(original_mm_items): # explicit None check, not `a or b`: the value is a multi-element # tensor whose truthiness is ambiguous.) image_grid_thw = item.model_specific_data.get("image_grid_thw") + if image_grid_thw is None: + image_grid_thw = item.model_specific_data.get("grid_thws") if image_grid_thw is None: image_grid_thw = item.model_specific_data.get("image_grid_hws") grid_len = _get_length(image_grid_thw) diff --git a/python/sglang/srt/managers/schedule_batch.py b/python/sglang/srt/managers/schedule_batch.py index 45ff08e36b..92c8ead68b 100755 --- a/python/sglang/srt/managers/schedule_batch.py +++ b/python/sglang/srt/managers/schedule_batch.py @@ -53,6 +53,7 @@ ScheduleBatch -> ForwardBatch import copy import dataclasses +import hashlib import logging import re import sys @@ -160,6 +161,10 @@ INIT_INCREMENTAL_DETOKENIZATION_OFFSET = 5 # This ensures pad_values don't overlap with valid text token IDs. MM_PAD_SHIFT_VALUE = 1_000_000 _MM_HASH_MASK = (1 << 64) - 1 +_MM_IDENTITY_BYTES = 32 +_MM_WIDE_PAD_TAG = b"sglang-mm-wide-pad-v1" +# Keep MM_PAD_SHIFT_VALUE + sentinel below 2**63 so it fits an int64 token ID. +_MM_WIDE_PAD_MASK = (1 << 62) - 1 logger = logging.getLogger(__name__) @@ -255,9 +260,15 @@ def split_cached_prefix_by_tier( return device, host, storage -def _compute_pad_value(hash: int) -> int: - """Compute pad value from hash.""" - return MM_PAD_SHIFT_VALUE + (hash % (1 << 30)) +def _compute_pad_values(identity: bytes) -> Tuple[int, ...]: + """derive shared wide sentinels from an item hash or full identity digest""" + digest = hashlib.sha256(_MM_WIDE_PAD_TAG + identity).digest() + # prefix matching can stop after one sentinel, so each must retain wide bits + return tuple( + MM_PAD_SHIFT_VALUE + + (int.from_bytes(digest[i : i + 8], byteorder="big") & _MM_WIDE_PAD_MASK) + for i in range(0, _MM_IDENTITY_BYTES, 8) + ) class BaseFinishReason: @@ -391,6 +402,11 @@ class MultimodalDataItem(msgspec.Struct, kw_only=True, dict=True, array_like=Tru default_factory=dict ) + # optional full identity; otherwise sentinels derive from the 64-bit item hash + # pad_value remains the first sentinel for scalar-padding consumers + identity: Optional[bytes] = None + pad_values: Optional[Tuple[int, ...]] = None + def __post_init__(self) -> None: if self.hash is not None: msgspec.Struct.__setattr__(self, "hash", self.hash & _MM_HASH_MASK) @@ -419,7 +435,33 @@ class MultimodalDataItem(msgspec.Struct, kw_only=True, dict=True, array_like=Tru def set_hash(self, hash_value: int) -> None: self.hash = hash_value - self.pad_value = _compute_pad_value(hash_value) + self.identity = None + self.pad_values = _compute_pad_values(self.hash.to_bytes(8, byteorder="big")) + self.pad_value = self.pad_values[0] + + def set_identity(self, identity: bytes) -> None: + """Adopt a full 32-byte digest and derive four 62-bit pad sentinels. + + ``hash`` keeps the leading 64 bits for scalar consumers such as the + embedding cache; the radix key uses the full sentinel pattern. + """ + if len(identity) != _MM_IDENTITY_BYTES: + raise ValueError( + f"multimodal identity must be {_MM_IDENTITY_BYTES} bytes, " + f"got {len(identity)}" + ) + self.identity = bytes(identity) + self.hash = int.from_bytes(identity[:8], byteorder="big", signed=False) + self.pad_values = _compute_pad_values(self.identity) + self.pad_value = self.pad_values[0] + + def padding_values(self) -> Tuple[int, ...]: + return self.pad_values or (self.pad_value,) + + def padding_sequence(self, length: int) -> List[int]: + values = self.padding_values() + repeats, remainder = divmod(length, len(values)) + return list(values) * repeats + list(values[:remainder]) @staticmethod def is_empty_list(l): @@ -433,12 +475,13 @@ class MultimodalDataItem(msgspec.Struct, kw_only=True, dict=True, array_like=Tru from sglang.srt.multimodal.cache import resolve_multimodal_item_hash - self.hash = resolve_multimodal_item_hash( - existing_hash=self.hash, - feature=self.feature, - precomputed_embeddings=self.precomputed_embeddings, + self.set_hash( + resolve_multimodal_item_hash( + existing_hash=self.hash, + feature=self.feature, + precomputed_embeddings=self.precomputed_embeddings, + ) ) - self.pad_value = _compute_pad_value(self.hash) def is_modality(self, modality: Modality) -> bool: return self.modality == modality @@ -650,7 +693,9 @@ class MultimodalProcessorOutput( for item in mm_items: for start, end in item.offsets: - padded_input_ids[start : end + 1] = [item.pad_value] * (end - start + 1) + padded_input_ids[start : end + 1] = item.padding_sequence( + end - start + 1 + ) return padded_input_ids @staticmethod diff --git a/python/sglang/srt/mem_cache/cpp_utils/hash_binding.cpp b/python/sglang/srt/mem_cache/cpp_utils/hash_binding.cpp index d415bdf781..15fc57a943 100644 --- a/python/sglang/srt/mem_cache/cpp_utils/hash_binding.cpp +++ b/python/sglang/srt/mem_cache/cpp_utils/hash_binding.cpp @@ -61,12 +61,18 @@ parse_prior_digest(py::object prior_digest_obj, bool *has_prior_digest) { inline void hash_page(const unsigned char *data, std::size_t len, bool &has_prior_digest, - std::array &prior_digest) { + std::array &prior_digest, + bool wide = false) { SHA256_CTX ctx; SHA256_Init(&ctx); if (has_prior_digest) { SHA256_Update(&ctx, prior_digest.data(), prior_digest.size()); } + if (wide) { + // an odd byte count separates wide encoding from legacy u32 pages + const unsigned char tag = 0xff; + SHA256_Update(&ctx, &tag, sizeof(tag)); + } if (len > 0) { SHA256_Update(&ctx, data, len); } @@ -74,6 +80,40 @@ inline void hash_page(const unsigned char *data, std::size_t len, has_prior_digest = true; } +template +bool has_wide_tokens(const RawToken *raw, std::size_t count) { + if constexpr (sizeof(RawToken) == 4) { + return false; + } + bool wide = false; + for (std::size_t i = 0; i < count; ++i) { + if (raw[i] > INT64_MAX) { + throw std::out_of_range("token id must be a non-negative int64"); + } + wide |= raw[i] > UINT32_MAX; + } + return wide; +} + +template +void hash_wide_page(const RawToken *raw, std::size_t count, bool is_bigram, + bool &has_prior_digest, + std::array &prior_digest) { + std::vector words; + if (is_bigram) { + words.reserve(count * 2); + for (std::size_t i = 0; i < count; ++i) { + words.push_back(raw[i]); + words.push_back(raw[i + 1]); + } + } else { + words.assign(raw, raw + count); + } + hash_page(reinterpret_cast(words.data()), + words.size() * sizeof(std::uint64_t), has_prior_digest, + prior_digest, true); +} + template inline void fill_regular_page(const RawToken *raw, std::size_t start, std::size_t count, std::uint32_t *out) { @@ -203,6 +243,16 @@ void hash_pages_to_hex_blob(const RawToken *raw, std::size_t logical_len, for (std::size_t start = 0, page_idx = 0; start < logical_len; start += page_size, ++page_idx) { const std::size_t page_units = std::min(page_size, logical_len - start); + const auto *page_start = raw + (is_bigram ? start : start * unit_width); + const std::size_t raw_count = + is_bigram ? page_units + 1 : page_units * unit_width; + if (has_wide_tokens(page_start, raw_count)) { + hash_wide_page(page_start, is_bigram ? page_units : raw_count, is_bigram, + has_prior_digest, prior_digest); + digest_to_hex_chars(prior_digest.data(), + hex_blob.data() + page_idx * kHexLen); + continue; + } const std::size_t page_bytes = page_units * unit_width * sizeof(std::uint32_t); const unsigned char *bytes = nullptr; @@ -233,6 +283,13 @@ std::string hash_all(const RawToken *raw, std::size_t logical_len, if (is_bigram) { unit_width = 2; } + const std::size_t raw_count = + is_bigram && logical_len > 0 ? logical_len + 1 : logical_len * unit_width; + if (has_wide_tokens(raw, raw_count)) { + hash_wide_page(raw, is_bigram ? logical_len : raw_count, is_bigram, + has_prior_digest, prior_digest); + return digest_to_hex_string(prior_digest.data()); + } const bool can_hash_raw_bytes = std::is_same_v && !is_bigram; const unsigned char *bytes = nullptr; diff --git a/python/sglang/srt/mem_cache/cpp_utils/native_hash.py b/python/sglang/srt/mem_cache/cpp_utils/native_hash.py index f23c1f708c..a9cac7b3b2 100644 --- a/python/sglang/srt/mem_cache/cpp_utils/native_hash.py +++ b/python/sglang/srt/mem_cache/cpp_utils/native_hash.py @@ -60,7 +60,7 @@ def _native_hash_input(token_ids: Any) -> tuple[array, int, int, bool]: return raw, logical_len, 2 if is_bigram else 1, is_bigram if is_bigram: - return array("I", raw[: logical_len + 1]), logical_len, 2, is_bigram + return array("q", raw[: logical_len + 1]), logical_len, 2, is_bigram if logical_len == 0: return array("I"), logical_len, 1, is_bigram @@ -69,13 +69,13 @@ def _native_hash_input(token_ids: Any) -> tuple[array, int, int, bool]: if isinstance(first_token, tuple): unit_width = len(first_token) return ( - array("I", (elem for token in raw[:logical_len] for elem in token)), + array("q", (elem for token in raw[:logical_len] for elem in token)), logical_len, unit_width, is_bigram, ) - return array("I", raw[:logical_len]), logical_len, 1, is_bigram + return array("q", raw[:logical_len]), logical_len, 1, is_bigram def get_native_hash( diff --git a/python/sglang/srt/models/ernie45_vl.py b/python/sglang/srt/models/ernie45_vl.py index 20ace5aa68..a3febfc820 100644 --- a/python/sglang/srt/models/ernie45_vl.py +++ b/python/sglang/srt/models/ernie45_vl.py @@ -652,12 +652,12 @@ class Ernie4_5_VLMoeForConditionalGeneration(nn.Module): ) pad_values = [] - if hasattr(forward_batch, "mm_inputs") and forward_batch.mm_inputs is not None: + if forward_batch.mm_inputs is not None: for mm_input in forward_batch.mm_inputs: if mm_input is None: continue for item in mm_input.mm_items: - pad_values.append(item.pad_value) + pad_values.extend(item.padding_values()) placeholder_tensor = torch.as_tensor( pad_values, device=input_ids.device, diff --git a/python/sglang/srt/multimodal/processors/kimi_common.py b/python/sglang/srt/multimodal/processors/kimi_common.py index f59c8f3fed..7d72e7977b 100644 --- a/python/sglang/srt/multimodal/processors/kimi_common.py +++ b/python/sglang/srt/multimodal/processors/kimi_common.py @@ -3,16 +3,72 @@ Shared by KimiVLImageProcessor and KimiK2_5VLImageProcessor. """ -from typing import Optional, Union +import hashlib +from collections.abc import Mapping +from dataclasses import dataclass +from typing import Any, List, Optional, Sequence, Union import numpy as np import torch +from sglang.srt.environ import envs +from sglang.srt.managers.io_struct import GenerateReqInput from sglang.srt.managers.schedule_batch import ( Modality, MultimodalDataItem, + MultimodalInputFormat, MultimodalProcessorOutput, ) +from sglang.srt.multimodal.cache import ( + CONTENT_HASH_PREFIX, + build_artifact_key, + build_processor_fingerprint, + parse_content_hash, + snapshot_media, +) + +_KIMI_IMAGE_IDENTITY_TAG = b"sglang-kimi-image-identity-v1" +_CALLER_IDENTITY_KEYS = ("hash", "pad_value", "pad_values", "identity") + + +@dataclass(frozen=True) +class KimiLoadedImage: + """Decoded image and the digest of the exact input bytes read by its loader.""" + + image: Any + content_digest: str + + +def unwrap_kimi_images(images): + return [ + image.image if isinstance(image, KimiLoadedImage) else image for image in images + ] + + +def kimi_image_identity(content_config_digest: str, grid_thw: Sequence[int]) -> bytes: + """Full SHA-256 image identity: content and preprocessing config, then grid.""" + digest = parse_content_hash(content_config_digest) + if digest is None: + raise ValueError("Kimi image identity requires a content digest") + hasher = hashlib.sha256(_KIMI_IMAGE_IDENTITY_TAG) + hasher.update(bytes.fromhex(digest[len(CONTENT_HASH_PREFIX) :])) + for dim in grid_thw: + hasher.update(int(dim).to_bytes(8, byteorder="big", signed=True)) + return hasher.digest() + + +def _item_grid_thw(item: MultimodalDataItem) -> List[int]: + grid = item.model_specific_data.get("image_grid_thw") + if grid is None: + grid = item.model_specific_data.get("grid_thws") + if grid is None: + raise ValueError("Kimi image item is missing image_grid_thw") + values = torch.as_tensor(grid).reshape(-1).tolist() + if len(values) == 2: + values.insert(0, 1) + if len(values) != 3: + raise ValueError(f"Kimi image item needs one [t, h, w] grid, got {values}") + return values class KimiGridMMDataMixin: @@ -23,6 +79,102 @@ class KimiGridMMDataMixin: - self._tokenizer (with .encode()) """ + # Opt-in: processors whose image spans use full identities and wide pads. + uses_wide_image_identity = False + + @classmethod + def _load_single_item(cls, data, modality, *args, **kwargs): + if ( + not cls.uses_wide_image_identity + or modality != Modality.IMAGE + or cls._is_preprocessed_input(data) + or envs.SGLANG_MM_SKIP_COMPUTE_HASH.get() + ): + return super()._load_single_item(data, modality, *args, **kwargs) + # hash the bytes already on the host, before nvJPEG creates a GPU tensor + snapshot = snapshot_media(data) + image = super()._load_single_item(snapshot.data, modality, *args, **kwargs) + return KimiLoadedImage(image, snapshot.content_digest) + + def process_mm_data(self, input_text, images=None, **kwargs): + return super().process_mm_data( + input_text, + images=unwrap_kimi_images(images) if images else images, + **kwargs, + ) + + @staticmethod + def reject_caller_image_identity(image_data, request_obj) -> None: + """Image identities must come from this processor, never from callers.""" + if isinstance(request_obj, GenerateReqInput) and request_obj.mm_hashes: + raise ValueError("Caller-supplied mm_hashes are not supported for Kimi.") + for item in image_data or []: + if isinstance(item, Mapping): + for key in _CALLER_IDENTITY_KEYS: + if key in item: + raise ValueError( + f"Caller-supplied multimodal {key} is not supported " + "for Kimi." + ) + + def _kimi_config_fingerprint(self) -> str: + fingerprint = self.processor_fingerprint + if fingerprint is None: + fingerprint = build_processor_fingerprint(self, self.hf_config) + self.processor_fingerprint = fingerprint + return fingerprint + + def kimi_content_config_digest(self, media: Any) -> str: + return build_artifact_key( + media.content_digest + if isinstance(media, KimiLoadedImage) + else snapshot_media(media).content_digest, + modality="image", + processor_fingerprint=self._kimi_config_fingerprint(), + ) + + def assign_kimi_image_identities( + self, mm_items: List[MultimodalDataItem], images: Optional[List[Any]] + ) -> None: + """Give each image item a full content+config+grid identity. + + Source images identify processor-computed items; preprocessed or + precomputed inputs are identified by their own tensor contents. + """ + if envs.SGLANG_MM_SKIP_COMPUTE_HASH.get(): + return + image_items = [item for item in mm_items if item.is_image()] + use_sources = ( + images is not None + and len(images) == len(image_items) + and all(item.format == MultimodalInputFormat.NORMAL for item in image_items) + ) + for index, item in enumerate(image_items): + if use_sources: + source = images[index] + elif item.feature is not None: + source = item.feature + else: + source = item.precomputed_embeddings + item.set_identity( + kimi_image_identity( + self.kimi_content_config_digest(source), _item_grid_thw(item) + ) + ) + + def _postprocess_mm_items_before_transport( + self, + mm_items: List[MultimodalDataItem], + *, + images: Optional[List[Any]], + ) -> List[MultimodalDataItem]: + mm_items = super()._postprocess_mm_items_before_transport( + mm_items, images=images + ) + if self.uses_wide_image_identity: + self.assign_kimi_image_identities(mm_items, images) + return mm_items + def resolve_image_token_counts(self, images): """Kimi's processor is remote-code and does not implement the transformers ``_get_num_multimodal_tokens`` convention; use its @@ -30,6 +182,7 @@ class KimiGridMMDataMixin: """ assert images is not None + images = unwrap_kimi_images(images) media_tokens_calculator = ( self._processor.media_processor.media_tokens_calculator ) @@ -125,7 +278,7 @@ class KimiGridMMDataMixin: image_embeddings = embeddings[Modality.IMAGE] mm_items = [] consumed = 0 - for start, end in offsets: + for (start, end), grid in zip(offsets, img_grid_thw): num_tokens = end - start + 1 embedding_slice = image_embeddings[consumed : consumed + num_tokens] consumed += num_tokens @@ -134,9 +287,13 @@ class KimiGridMMDataMixin: modality=Modality.IMAGE, offsets=[(start, end)], precomputed_embeddings=embedding_slice, + model_specific_data={"image_grid_thw": grid}, ) ) + if self.uses_wide_image_identity: + self.assign_kimi_image_identities(mm_items, None) + return MultimodalProcessorOutput( input_ids=input_ids, mm_items=mm_items, diff --git a/python/sglang/srt/multimodal/processors/kimi_k25.py b/python/sglang/srt/multimodal/processors/kimi_k25.py index b477589f28..727c853e7a 100644 --- a/python/sglang/srt/multimodal/processors/kimi_k25.py +++ b/python/sglang/srt/multimodal/processors/kimi_k25.py @@ -10,9 +10,7 @@ from PIL import Image from sglang.kernels.ops.mm.process import normalize_and_patchify from sglang.srt.environ import envs -from sglang.srt.managers.schedule_batch import ( - MultimodalProcessorOutput, -) +from sglang.srt.managers.schedule_batch import MultimodalProcessorOutput from sglang.srt.models.kimi_k25 import KimiK25ForConditionalGeneration from sglang.srt.multimodal.processors.base_processor import ( BaseMultimodalProcessor as SGLangBaseProcessor, @@ -387,6 +385,20 @@ class KimiGPUProcessorWrapper: self.tokenizer = hf_processor.tokenizer self.media_processor = hf_processor.media_processor + def preprocess_fingerprint_payload(self): + return { + "patch_size": self._patch_size, + "merge_kernel_size": self._merge_kernel_size, + "in_patch_limit": self._in_patch_limit, + "patch_limit_on_one_side": self._patch_limit_on_one_side, + "fixed_output_tokens": self._fixed_output_tokens, + "image_mean": self._image_mean, + "image_std": self._image_std, + "hf_media_config": self.media_processor.media_proc_cfg, + "force_cpu": _FORCE_CPU_IMAGE_PREPROCESSING, + "cuda_available": torch.cuda.is_available(), + } + def __call__(self, text=None, images=None, **kwargs): # process_mm_data passes images via kwargs["images"] images = images or kwargs.pop("images", None) @@ -521,6 +533,7 @@ class KimiK2_5VLImageProcessor(KimiGridMMDataMixin, SGLangBaseProcessor): auto_mm_processor_worker_num = 2 auto_mm_io_worker_num = 16 supports_mm_processor_concurrency = True + uses_wide_image_identity = True def __init__(self, hf_config, server_args, _processor, *args, **kwargs): mm_tokens = MultimodalSpecialTokens( @@ -556,6 +569,7 @@ class KimiK2_5VLImageProcessor(KimiGridMMDataMixin, SGLangBaseProcessor): *args, **kwargs, ): + self.reject_caller_image_identity(image_data, request_obj) expected_image_count = len(image_data or []) placeholder_count = self.count_image_placeholders( input_text, self.mm_tokens.image_token_id diff --git a/python/sglang/srt/multimodal/processors/kimi_k3.py b/python/sglang/srt/multimodal/processors/kimi_k3.py index 1dfa42408c..dbd3df5cd3 100644 --- a/python/sglang/srt/multimodal/processors/kimi_k3.py +++ b/python/sglang/srt/multimodal/processors/kimi_k3.py @@ -17,13 +17,14 @@ import numpy as np import torch from PIL import Image +from sglang.srt.environ import envs from sglang.srt.managers.schedule_batch import ( Modality, MultimodalDataItem, MultimodalProcessorOutput, ) from sglang.srt.models.kimi_k3 import KimiK3ForConditionalGeneration -from sglang.srt.multimodal.cache import resolve_multimodal_item_hash +from sglang.srt.multimodal.cache import resolve_multimodal_item_hash, snapshot_media from sglang.srt.multimodal.kimi_k3_image_processing import ( DEFERRED_PREPROCESSING_KEY, KimiK3DeferredPreprocessing, @@ -49,7 +50,12 @@ from sglang.srt.multimodal.processors.base_processor import ( from sglang.srt.multimodal.processors.base_processor import ( MultimodalSpecialTokens, ) -from sglang.srt.multimodal.processors.kimi_common import KimiGridMMDataMixin +from sglang.srt.multimodal.processors.kimi_common import ( + KimiGridMMDataMixin, + KimiLoadedImage, + kimi_image_identity, + unwrap_kimi_images, +) from sglang.srt.multimodal.processors.kimi_k25 import ( KimiGPUProcessorWrapper, _get_image_dimensions, @@ -381,6 +387,7 @@ class KimiK3ImageProcessor( auto_mm_preprocess_cache_size_mb = 256 supports_mm_processor_concurrency = True preserve_processor_input_ids = True + uses_wide_image_identity = True def __init__(self, hf_config, server_args, _processor, *args, **kwargs): mm_tokens = MultimodalSpecialTokens( @@ -406,6 +413,7 @@ class KimiK3ImageProcessor( """ when raw_bytes <= processed_bytes, preprocess first would introduce larger payload, so deferring gpu preprocessing would benefit """ + images = unwrap_kimi_images(images or []) if ( not images or self.mm_feature_transport != "cpu" @@ -452,13 +460,14 @@ class KimiK3ImageProcessor( return raw_bytes <= processed_bytes def _build_deferred_output(self, base_output): + images = unwrap_kimi_images(base_output.images) ( input_ids, resize_configs, deferred_preprocessing, ) = self._processor.prepare_deferred( base_output.input_text, - base_output.images, + images, base_output.input_ids, ) offsets = self.get_mm_items_offset( @@ -468,9 +477,7 @@ class KimiK3ImageProcessor( raise ValueError("Expected one Kimi-K3 image span for each image") items = [] - for image, resize_config, offset in zip( - base_output.images, resize_configs, offsets - ): + for image, resize_config, offset in zip(images, resize_configs, offsets): grid_thw = _grid_thw_from_resize_config( resize_config, self._processor.preprocess_config.patch_size ) @@ -487,6 +494,7 @@ class KimiK3ImageProcessor( ) items.append(item) + self.assign_kimi_image_identities(items, base_output.images) self._precompute_hashes_before_cpu_transfer(items) return MultimodalProcessorOutput( input_ids=input_ids.flatten().tolist(), @@ -644,7 +652,12 @@ class KimiK3ImageProcessor( offsets=[offset], model_specific_data=model_specific_data, ) - item.set_hash(artifact.feature_hash) + if envs.SGLANG_MM_SKIP_COMPUTE_HASH.get(): + item.set_hash(artifact.feature_hash) + else: + item.set_identity( + kimi_image_identity(artifact.artifact_key, artifact.grid_thw) + ) if self.keep_mm_features_on_device and item.feature is not None: item.model_specific_data[DEFER_CUDA_IPC_FEATURE_RECONSTRUCTION_KEY] = ( True @@ -703,6 +716,17 @@ class KimiK3ImageProcessor( im_token_id=self.mm_tokens.image_token_id, ) + def kimi_content_config_digest(self, media) -> str: + self._kimi_config_fingerprint() + digest = ( + media.content_digest + if isinstance(media, KimiLoadedImage) + else snapshot_media(media).content_digest + ) + return self._artifact_key( + digest, media.image if isinstance(media, KimiLoadedImage) else media + ) + async def process_mm_data_async( self, image_data: List[Union[str, bytes, Dict]], @@ -713,6 +737,7 @@ class KimiK3ImageProcessor( ): if request_obj.video_data or kwargs.get("audio_data"): raise ValueError("Kimi-K3 supports image input only") + self.reject_caller_image_identity(image_data, request_obj) expected_image_count = len(image_data or []) placeholder_count = self.count_image_placeholders( diff --git a/python/sglang/srt/speculative/ngram_worker.py b/python/sglang/srt/speculative/ngram_worker.py index 2517c0d65a..fa80e19554 100644 --- a/python/sglang/srt/speculative/ngram_worker.py +++ b/python/sglang/srt/speculative/ngram_worker.py @@ -252,14 +252,17 @@ class NGRAMWorker(BaseSpecWorker): req_ids = [req.rid for req in batch.reqs] # Only the last max_trie_depth tokens can match; the ids are # array.array, so list() the tails for list concat below. + # multimodal cache padding contains identities, not vocabulary token IDs input_tails = [ - list(req.origin_input_ids[-self.max_trie_depth :]) for req in batch.reqs + list(req.origin_input_ids_unpadded[-self.max_trie_depth :]) + for req in batch.reqs ] output_tails = [ list(req.output_ids[-self.max_trie_depth :]) for req in batch.reqs ] base_lens = [ - len(req.origin_input_ids) + len(req.output_ids) for req in batch.reqs + len(req.origin_input_ids_unpadded) + len(req.output_ids) + for req in batch.reqs ] if use_prev_tokens: @@ -414,7 +417,7 @@ class NGRAMWorker(BaseSpecWorker): else [] ) put_ids = self._efficient_concat_last_n( - list(req.origin_input_ids[-self.max_trie_depth :]), + list(req.origin_input_ids_unpadded[-self.max_trie_depth :]), list(req.output_ids[-self.max_trie_depth :]) + prev_tokens, self.max_trie_depth, ) diff --git a/rust/sglang-radix-tree/src/node.rs b/rust/sglang-radix-tree/src/node.rs index 7c6fecd631..4c0b563307 100644 --- a/rust/sglang-radix-tree/src/node.rs +++ b/rust/sglang-radix-tree/src/node.rs @@ -836,8 +836,8 @@ pub trait ChildKeyType: /// through, so the unigram key never copies. fn key_from(token_ids: Cow<'_, Vec>) -> Cow<'_, Self>; - /// The atom's token ids as u32 storage-hash words. - fn hash_words(atom: &Self::Atom) -> impl Iterator; + /// The atom's token ids as non-negative storage-hash words. + fn hash_words(atom: &Self::Atom) -> impl Iterator; /// The raw token ids spanned by `atoms`; the unigram view borrows, bigram /// atoms (overlapping by one) materialize. @@ -915,9 +915,9 @@ pub trait ChildKeyType: } } -/// A token id as a u32 storage-hash word; token ids beyond u32 are rejected. -fn hash_word(token_id: i64) -> u32 { - u32::try_from(token_id).expect("token id does not fit in uint32") +/// Wide multimodal sentinels use the same non-negative int64 domain as tree keys. +fn hash_word(token_id: i64) -> u64 { + u64::try_from(token_id).expect("token id must be a non-negative int64") } impl ChildKeyType for Vec { @@ -928,7 +928,7 @@ impl ChildKeyType for Vec { token_ids } - fn hash_words(atom: &i64) -> impl Iterator { + fn hash_words(atom: &i64) -> impl Iterator { std::iter::once(hash_word(*atom)) } @@ -946,7 +946,7 @@ impl ChildKeyType for Vec<(i64, i64)> { Cow::Owned(token_ids.windows(2).map(|w| (w[0], w[1])).collect()) } - fn hash_words(atom: &(i64, i64)) -> impl Iterator { + fn hash_words(atom: &(i64, i64)) -> impl Iterator { [hash_word(atom.0), hash_word(atom.1)].into_iter() } @@ -967,7 +967,7 @@ impl ChildKeyType for Vec<(i64, i64)> { pub(crate) const DIGEST_LEN: usize = 32; pub(crate) type HashDigest = [u8; DIGEST_LEN]; -/// SHA256(prior_digest || page atom words as little-endian u32 bytes). +/// Preserve u32 page hashes; wide pages use a 0xff tag and little-endian u64 words. pub(crate) fn hash_page( page: &[K::Atom], prior: Option<&HashDigest>, @@ -976,9 +976,21 @@ pub(crate) fn hash_page( if let Some(prior) = prior { hasher.update(prior); } + let wide = page + .iter() + .flat_map(K::hash_words) + .any(|word| word > u32::MAX as u64); + if wide { + // match hash_binding.cpp; the tag makes the encoding length odd + hasher.update([0xff]); + } for atom in page { for word in K::hash_words(atom) { - hasher.update(word.to_le_bytes()); + if wide { + hasher.update(word.to_le_bytes()); + } else { + hasher.update((word as u32).to_le_bytes()); + } } } hasher.finalize().into() diff --git a/rust/sglang-radix-tree/src/python_bindings.rs b/rust/sglang-radix-tree/src/python_bindings.rs index d0ff21f63a..879dc7ee6c 100644 --- a/rust/sglang-radix-tree/src/python_bindings.rs +++ b/rust/sglang-radix-tree/src/python_bindings.rs @@ -4019,12 +4019,9 @@ fn get_hash_str( "prior_hash must be a 64-character hexadecimal digest", )); } - if let Some(token_id) = raw - .iter() - .find(|token_id| u32::try_from(**token_id).is_err()) - { + if let Some(token_id) = raw.iter().find(|token_id| **token_id < 0) { return Err(PyValueError::new_err(format!( - "token id {token_id} does not fit in uint32" + "token id {token_id} must be a non-negative int64" ))); } Ok(py.detach(move || { diff --git a/rust/sglang-radix-tree/src/tests/node.rs b/rust/sglang-radix-tree/src/tests/node.rs index 23cb4a7812..c5ec7de3b6 100644 --- a/rust/sglang-radix-tree/src/tests/node.rs +++ b/rust/sglang-radix-tree/src/tests/node.rs @@ -54,8 +54,8 @@ impl ChildKeyType for CountedKey { Cow::Owned(Self::from(token_ids.into_owned())) } - fn hash_words(atom: &Self::Atom) -> impl Iterator { - std::iter::once(*atom as u32) + fn hash_words(atom: &Self::Atom) -> impl Iterator { + std::iter::once(*atom as u64) } fn raw_token_ids(atoms: &[Self::Atom]) -> Cow<'_, [i64]> { @@ -880,13 +880,15 @@ fn empty_key_yields_no_pages() { } #[test] -#[should_panic(expected = "token id does not fit in uint32")] -fn oversized_token_id_is_rejected() { - get_hash_str::>(&[1 << 33], None, 2); +fn wide_token_ids_keep_their_high_bits() { + assert_ne!( + get_hash_str::>(&[1 << 33], None, 2), + get_hash_str::>(&[0], None, 2) + ); } #[test] -#[should_panic(expected = "token id does not fit in uint32")] +#[should_panic(expected = "token id must be a non-negative int64")] fn negative_token_id_is_rejected() { get_hash_str::>(&[-1], None, 2); } diff --git a/test/registered/unit/managers/test_mm_hashes.py b/test/registered/unit/managers/test_mm_hashes.py index 5247fa4ac9..85d92ea2e1 100644 --- a/test/registered/unit/managers/test_mm_hashes.py +++ b/test/registered/unit/managers/test_mm_hashes.py @@ -26,9 +26,9 @@ from sglang.srt.environ import envs from sglang.srt.managers.io_struct import GenerateReqInput from sglang.srt.managers.mm_utils import hash_feature from sglang.srt.managers.schedule_batch import ( + MM_PAD_SHIFT_VALUE, Modality, MultimodalDataItem, - _compute_pad_value, ) from sglang.srt.multimodal.processors.hash_executor import MultimodalHashExecutor from sglang.test.ci.ci_register import register_cpu_ci @@ -66,7 +66,8 @@ class TestMmHashesContract(CustomTestCase): ): item.set_pad_value() self.assertEqual(item.hash, 0xDEADBEEF) - self.assertEqual(item.pad_value, _compute_pad_value(0xDEADBEEF)) + self.assertEqual(len(item.padding_values()), 4) + self.assertEqual(item.pad_value, item.padding_values()[0]) def test_set_pad_value_is_deterministic_across_items(self): """Two items with the same preset hash must derive the same pad_value.""" @@ -75,7 +76,7 @@ class TestMmHashesContract(CustomTestCase): # No feature payload — set_pad_value uses the preset hash. a.set_pad_value() b.set_pad_value() - self.assertEqual(a.pad_value, b.pad_value) + self.assertEqual(a.padding_values(), b.padding_values()) self.assertEqual(a.hash, b.hash) def test_set_pad_value_distinguishes_different_preset_hashes(self): @@ -93,7 +94,41 @@ class TestMmHashesContract(CustomTestCase): item.set_hash(0xBBBB) self.assertEqual(item.hash, 0xBBBB) - self.assertEqual(item.pad_value, _compute_pad_value(0xBBBB)) + expected = MultimodalDataItem(modality=Modality.IMAGE, hash=0xBBBB) + expected.set_pad_value() + self.assertEqual(item.padding_values(), expected.padding_values()) + + def test_wide_padding_is_shared_by_all_modalities(self): + expected = ( + 4445679232235744898, + 1252851680943950380, + 4490111419971165939, + 1999168716371743145, + ) + for modality in Modality.all(): + with self.subTest(modality=modality): + a = MultimodalDataItem(modality=modality, hash=7) + b = MultimodalDataItem(modality=modality, hash=7 + (1 << 30)) + a.set_pad_value() + b.set_pad_value() + self.assertNotEqual(a.pad_value, b.pad_value) + self.assertEqual(a.padding_values(), expected) + self.assertEqual( + a.padding_sequence(6), list(a.pad_values) + list(a.pad_values[:2]) + ) + for value in a.padding_values(): + self.assertGreaterEqual(value, MM_PAD_SHIFT_VALUE) + self.assertLess(value, 1 << 63) + + def test_set_hash_and_constructor_normalize_identically(self): + for value in (-1, 1 << 64, (1 << 128) + 7): + with self.subTest(value=value): + a = MultimodalDataItem(modality=Modality.IMAGE, hash=value) + b = MultimodalDataItem(modality=Modality.IMAGE) + a.set_pad_value() + b.set_hash(value) + self.assertEqual(a.hash, b.hash) + self.assertEqual(a.padding_values(), b.padding_values()) class TestMultimodalHashExecutor(unittest.IsolatedAsyncioTestCase, CustomTestCase): @@ -127,8 +162,8 @@ class TestMultimodalHashExecutor(unittest.IsolatedAsyncioTestCase, CustomTestCas ): received.set_pad_value() self.assertEqual( - (received.hash, received.pad_value), - (expected.hash, expected.pad_value), + (received.hash, received.padding_values()), + (expected.hash, expected.padding_values()), ) async def test_overrides_and_device_features_stay_on_caller_thread(self): diff --git a/test/registered/unit/managers/test_mm_utils_split.py b/test/registered/unit/managers/test_mm_utils_split.py index dd14559a87..935f9de55f 100644 --- a/test/registered/unit/managers/test_mm_utils_split.py +++ b/test/registered/unit/managers/test_mm_utils_split.py @@ -15,6 +15,7 @@ import unittest import warnings from array import array from functools import partial +from types import SimpleNamespace import numpy as np import torch @@ -24,6 +25,7 @@ from sglang.srt.managers.mm_utils import ( MultiModalityDataPaddingPatternMultimodalTokens, MultiModalityDataPaddingPatternTokenPairs, duplicate_pad_mm_input_ids, + embed_mm_inputs, get_new_expanded_mm_items, pad_mm_token_pairs, pad_mm_tokens, @@ -33,6 +35,7 @@ from sglang.srt.managers.schedule_batch import ( MultimodalDataItem, MultimodalInputs, ) +from sglang.srt.models.ernie45_vl import Ernie4_5_VLMoeForConditionalGeneration from sglang.test.ci.ci_register import register_cpu_ci from sglang.test.test_utils import CustomTestCase @@ -157,6 +160,58 @@ class TestGetNewExpandedMMItems(CustomTestCase): class TestMMPadding(CustomTestCase): + def test_ernie_visual_mask_recognizes_every_sentinel(self): + item = MultimodalDataItem(modality=Modality.IMAGE, hash=7, offsets=[(1, 6)]) + item.set_pad_value() + mm_inputs = MultimodalInputs(mm_items=[item], im_token_id=9) + padded = pad_mm_tokens(array("q", [1] + [9] * 6 + [2]), mm_inputs) + model = SimpleNamespace(_visual_token_ids_tensor_cache=torch.tensor([9])) + batch = SimpleNamespace(mm_inputs=[None, mm_inputs]) + Ernie4_5_VLMoeForConditionalGeneration._set_visual_token_mask( + model, torch.tensor(padded), batch + ) + self.assertEqual( + model.visual_token_mask.flatten().tolist(), [False] + [True] * 6 + [False] + ) + + def test_generic_padding_reaches_embedding_scatter(self): + devices = ["cpu"] + (["cuda"] if torch.cuda.is_available() else []) + for device in devices: + for modality in Modality.all(): + with self.subTest(device=device, modality=modality): + item = MultimodalDataItem( + modality=modality, + hash=7, + offsets=[(1, 6)], + precomputed_embeddings=torch.arange(18, device=device) + .reshape(6, 3) + .float(), + ) + item.set_pad_value() + mm_inputs = MultimodalInputs( + mm_items=[item], + im_token_id=9, + audio_token_id=9, + video_token_id=9, + ) + padded = pad_mm_tokens(array("q", [1] + [9] * 6 + [2]), mm_inputs) + embedding = torch.nn.Embedding(10, 3, device=device) + # chunked prefill may begin at any of the four sentinels + for prefix in (0, 1, 2, 3, 4): + ids = torch.tensor(padded[prefix:7], device=device) + actual, _ = embed_mm_inputs( + [mm_inputs], + [prefix], + [len(ids)], + ids, + embedding, + data_embedding_func_mapping={modality: lambda _: None}, + ) + expected = item.precomputed_embeddings[max(0, prefix - 1) :] + self.assertTrue( + torch.equal(actual[1:] if prefix == 0 else actual, expected) + ) + def test_padding_requires_int64_arrays(self): mm_inputs = MultimodalInputs(mm_items=[]) for pad in ( diff --git a/test/registered/unit/mem_cache/test_mem_cache_utils.py b/test/registered/unit/mem_cache/test_mem_cache_utils.py index 7fa7c35720..5735f1f5b3 100644 --- a/test/registered/unit/mem_cache/test_mem_cache_utils.py +++ b/test/registered/unit/mem_cache/test_mem_cache_utils.py @@ -200,6 +200,27 @@ class TestMaybeInitCustomMemPool(unittest.TestCase): class TestGetHashStr(unittest.TestCase): + def test_wide_tokens_preserve_page_chaining(self): + wide = (1 << 62) + 123 + tokens = [1, 2, wide, 4, 5] + first = _legacy_get_hash_str(tokens[:2]) + second = hashlib.sha256( + bytes.fromhex(first) + + b"\xff" + + wide.to_bytes(8, "little") + + (4).to_bytes(8, "little") + ).hexdigest() + expected = [first, second, _legacy_get_hash_str([5], second)] + for values in (tokens, array("q", tokens), array("Q", tokens)): + self.assertEqual(get_hash_str(values, page_size=2), expected) + self.assertEqual(get_hash_str(values[2:4], first), second) + self.assertNotEqual(get_hash_str([wide]), get_hash_str([wide & 0xFFFFFFFF])) + self.assertEqual(get_hash_str([(1, wide)]), get_hash_str([1, wide])) + self.assertEqual( + get_hash_str(_HashKey(array("q", [1, wide, 2]), is_bigram=True)), + get_hash_str([(1, wide), (wide, 2)]), + ) + def test_hash_str_matches_pre_optimization_per_token_loop(self): for name, tokens, prior_hash in _single_hash_compatibility_cases(): with self.subTest(name=name): diff --git a/test/registered/unit/mem_cache/test_rust_tree_core_integration.py b/test/registered/unit/mem_cache/test_rust_tree_core_integration.py index a5c03c9c19..bd16943df1 100644 --- a/test/registered/unit/mem_cache/test_rust_tree_core_integration.py +++ b/test/registered/unit/mem_cache/test_rust_tree_core_integration.py @@ -24,7 +24,13 @@ from sglang.srt.disaggregation.kv_events import ( namespaced_block_hash, ) from sglang.srt.environ import envs -from sglang.srt.managers.schedule_batch import Req +from sglang.srt.managers.mm_utils import pad_mm_tokens +from sglang.srt.managers.schedule_batch import ( + Modality, + MultimodalDataItem, + MultimodalInputs, + Req, +) from sglang.srt.mem_cache.allocator.token import TokenToKVPoolAllocator from sglang.srt.mem_cache.base_prefix_cache import ( DecLockRefParams, @@ -68,7 +74,11 @@ from sglang.srt.mem_cache.unified_cache.unified_tree_core_interface import ( UnifiedTreeCoreInterface, ) from sglang.srt.mem_cache.unified_radix_cache import UnifiedRadixCache -from sglang.srt.mem_cache.utils import get_storage_hash_str, hash_str_to_int64 +from sglang.srt.mem_cache.utils import ( + get_hash_str, + get_storage_hash_str, + hash_str_to_int64, +) from sglang.srt.runtime_context import get_context from sglang.srt.sampling.sampling_params import SamplingParams @@ -156,6 +166,45 @@ def test_match_on_the_empty_tree_returns_no_indices(): assert result.device_prefix_len == 0 +@pytest.mark.parametrize("backend", ["rust", "python"]) +@pytest.mark.parametrize("page_size", [1, 16]) +@pytest.mark.parametrize("image_len", [1, 2, 3, 8]) +@pytest.mark.parametrize("prefix_len", [0, 13, 14, 15, 16]) +def test_multimodal_hash_collision_does_not_reuse_image_kv( + backend, page_size, image_len, prefix_len +): + core = _tree_core(backend=backend, page_size=page_size) + # identical low 30 bits used to alias every image token + keys = [] + for item_hash in (7, 7 + (1 << 30)): + item = MultimodalDataItem( + modality=Modality.IMAGE, + hash=item_hash, + offsets=[(prefix_len, prefix_len + image_len - 1)], + ) + item.set_pad_value() + tokens = array("q", [1] * prefix_len + [9] * image_len + [2] * 32) + keys.append( + list( + pad_mm_tokens(tokens, MultimodalInputs(mm_items=[item], im_token_id=9)) + ) + ) + cached_len = len(keys[0]) // page_size * page_size + _insert(core, keys[0], list(range(cached_len))) + + same = core.match_prefix(MatchPrefixParams(key=_key(keys[0]))) + assert same.device_prefix_len == cached_len + assert _path_indices(core, same.last_device_node).tolist() == list( + range(cached_len) + ) + different = core.match_prefix(MatchPrefixParams(key=_key(keys[1]))) + text_hit_len = prefix_len // page_size * page_size + assert different.device_prefix_len == text_hit_len + assert _path_indices(core, different.last_device_node).tolist() == list( + range(text_hit_len) + ) + + @pytest.mark.parametrize("instance_backend", [None, "rust"]) def test_default_backend_constructs_real_rust_cpu_cache(monkeypatch, instance_backend): from sglang.srt.mem_cache.unified_radix_cache import UnifiedRadixCache @@ -1283,10 +1332,18 @@ def test_hash_boundary_rejects_malformed_prior_hash(prior_hash): mem_cache.get_hash_str(array("q", [1, 2]), prior_hash, 2) -@pytest.mark.parametrize("token_id", [-1, 1 << 32]) -def test_hash_boundary_rejects_token_ids_outside_uint32(token_id): - with pytest.raises(ValueError, match="does not fit in uint32"): - mem_cache.get_hash_str(array("q", [token_id]), None, 1) +def test_hash_boundary_rejects_negative_token_ids(): + with pytest.raises(ValueError, match="non-negative int64"): + mem_cache.get_hash_str(array("q", [-1]), None, 1) + + +@pytest.mark.parametrize("is_bigram", [False, True]) +def test_hash_boundary_preserves_wide_multimodal_tokens(is_bigram): + raw = array("q", [1, 2, (1 << 62) + 123, 4, 5]) + key = RadixKey(raw, is_bigram=is_bigram) + assert mem_cache.get_hash_str(raw, None, 2, is_bigram) == get_hash_str( + key, page_size=2 + ) def test_hash_boundary_rejects_zero_page_size(): diff --git a/test/registered/unit/models/test_kimi_k25.py b/test/registered/unit/models/test_kimi_k25.py index 4d79380bf5..f1993c9b42 100644 --- a/test/registered/unit/models/test_kimi_k25.py +++ b/test/registered/unit/models/test_kimi_k25.py @@ -33,6 +33,7 @@ from sglang.srt.models.kimi_k25 import ( from sglang.srt.models.kimi_vl_moonvit import tpool_patch_merger from sglang.srt.multimodal.cache import ( MultimodalPreprocessCache, + build_processor_fingerprint, resolve_multimodal_item_hash, snapshot_media, ) @@ -47,7 +48,10 @@ from sglang.srt.multimodal.media_artifacts.kimi_k3 import ( ) from sglang.srt.multimodal.mm_utils import run_dp_sharded_mrope_vision_model from sglang.srt.multimodal.processors.base_processor import BaseMultimodalProcessor -from sglang.srt.multimodal.processors.kimi_common import KimiGridMMDataMixin +from sglang.srt.multimodal.processors.kimi_common import ( + KimiGridMMDataMixin, + kimi_image_identity, +) from sglang.srt.multimodal.processors.kimi_k3 import ( KimiK3GPUProcessorWrapper, KimiK3ImageProcessor, @@ -66,12 +70,7 @@ from sglang.srt.multimodal.transport.cuda_ipc import ( DEFER_CUDA_IPC_FEATURE_RECONSTRUCTION_KEY, CudaIpcTensorTransportProxy, ) -from sglang.srt.runtime_context import ( - get_context, - get_parallel, - publish, - reset_context, -) +from sglang.srt.runtime_context import get_context, get_parallel, publish, reset_context from sglang.srt.server_args import ServerArgs from sglang.srt.utils import ImageData from sglang.test.ci.ci_register import register_cpu_ci @@ -638,6 +637,32 @@ def _k3_preprocess_config( ) +def test_kimi_k25_fingerprint_includes_gpu_preprocessing_options(): + fingerprints = [] + for mean in ([0.5, 0.5, 0.5], [0.1, 0.1, 0.1]): + wrapper = KimiGPUProcessorWrapper( + _HFProcessor(), + "<|media_pad|>", + 42, + 14, + 2, + 16384, + 512, + None, + mean, + [0.5, 0.5, 0.5], + ) + with get_context().override_server_args( + revision=None, disable_fast_image_processor=False, mm_process_config={} + ): + fingerprints.append( + build_processor_fingerprint( + wrapper, SimpleNamespace(to_dict=lambda: {}) + ) + ) + assert fingerprints[0] != fingerprints[1] + + @pytest.mark.parametrize( ("processor_cls", "wrapper_cls"), [ @@ -650,6 +675,9 @@ def test_kimi_processor_workers_clone_the_gpu_wrapper(processor_cls, wrapper_cls base_gpu_id=0, rl_on_policy_target=None, tp_size=1, + kv_events_config=None, + hicache_storage_backend=None, + disaggregation_mode="null", ) with get_context().override_server_args( mm_feature_transport="cpu", @@ -720,6 +748,7 @@ def test_kimi_k3_cpu_prompt_uses_the_same_media_contract(): def test_kimi_k3_epd_rebuild_uses_the_same_media_contract(): processor = object.__new__(KimiK3ImageProcessor) + processor.processor_fingerprint = "epd-test-config" processor.hf_config = SimpleNamespace( vision_config=SimpleNamespace(merge_kernel_size=(2, 2)) ) @@ -783,7 +812,7 @@ def test_kimi_k3_cached_artifact_is_composed_per_prompt(): processor._tokenizer = _Tokenizer() processor.mm_feature_transport = "cpu" processor.use_cuda_ipc = False - artifact = _cached_k3_artifact("sha256:" + "ab" * 32, "artifact") + artifact = _cached_k3_artifact("sha256:" + "ab" * 32, "sha256:" + "cd" * 32) first = processor.compose_request([1, 99, 2], [artifact]) second = processor.compose_request([3, 4, 99, 5], [artifact]) @@ -791,7 +820,10 @@ def test_kimi_k3_cached_artifact_is_composed_per_prompt(): assert first.input_ids != second.input_ids assert first.mm_items[0].offsets == [(3, 5)] assert second.mm_items[0].offsets == [(4, 6)] - assert first.mm_items[0].hash == second.mm_items[0].hash == 123 + identity = kimi_image_identity(artifact.artifact_key, artifact.grid_thw) + assert first.mm_items[0].identity == second.mm_items[0].identity == identity + assert first.mm_items[0].hash == second.mm_items[0].hash + assert first.mm_items[0].pad_values == second.mm_items[0].pad_values torch.testing.assert_close(first.mm_items[0].feature, second.mm_items[0].feature) @@ -1107,10 +1139,59 @@ def test_kimi_k3_artifact_and_data_item_share_hash_resolution(): existing_hash=direct_item.hash, namespace=artifact.artifact_key, ) - expected_item = MultimodalDataItem(modality=Modality.IMAGE, hash=expected_hash) - expected_item.set_pad_value() - assert artifact.feature_hash == composed_item.hash == expected_hash - assert composed_item.pad_value == expected_item.pad_value + assert artifact.feature_hash == expected_hash + expected_item = MultimodalDataItem(modality=Modality.IMAGE) + expected_item.set_identity( + kimi_image_identity(artifact.artifact_key, artifact.grid_thw) + ) + assert composed_item.identity == expected_item.identity + assert composed_item.hash == expected_item.hash + assert composed_item.pad_values == expected_item.pad_values + + # A freshly preprocessed image and the same image served from the + # preprocess cache must resolve to the same identity. + image = torch.arange(3 * 2 * 4, dtype=torch.uint8).reshape(3, 2, 4) + resize_config = { + "num_tokens": 2, + "new_width": 4, + "new_height": 2, + "pad_width": 0, + "pad_height": 2, + } + deferred = functools.partial( + KimiK3DeferredPreprocessing, + backend="gpu", + image_mean=[0.5, 0.5, 0.5], + image_std=[0.5, 0.5, 0.5], + transparent_bg_config=None, + ) + processor._processor = SimpleNamespace( + preprocess_config=_k3_preprocess_config(patch_size=2), + prepare_deferred=Mock( + return_value=(torch.tensor([[1, 99, 99, 2]]), [resize_config], deferred) + ), + ) + processor.processor_fingerprint = "sha256:" + "ef" * 32 + fresh_item = processor._build_deferred_output( + SimpleNamespace(input_text="prompt", images=[image], input_ids=[1, 99, 2]) + ).mm_items[0] + + cached_artifact = processor._make_artifact( + content_digest=snapshot_media(image).content_digest, + artifact_key=processor._artifact_key( + snapshot_media(image).content_digest, image + ), + original_size=(4, 2), + resize_config=resize_config, + grid_thw=tuple(fresh_item.image_grid_thw.flatten().tolist()), + feature=fresh_item.feature, + deferred=deferred(resize_config=resize_config), + ) + cached_item = processor.compose_request([1, 99, 2], [cached_artifact]).mm_items[0] + assert fresh_item.identity is not None + assert cached_item.identity == fresh_item.identity + assert cached_item.hash == fresh_item.hash + assert cached_item.pad_values == fresh_item.pad_values def test_kimi_k3_untrusted_path_change_is_a_cache_miss(): @@ -1294,10 +1375,15 @@ def test_kimi_k3_cpu_transport_defers_gpu_preprocessing(): input_text="prompt", images=images, input_ids=[1, 99, 2, 99, 3] ) + processor.processor_fingerprint = "sha256:" + "ef" * 32 + output = processor._build_deferred_output(base_output) assert output.input_ids == [1, 99, 99, 2, 99, 3] assert [item.offsets for item in output.mm_items] == [[(1, 2)], [(4, 4)]] + first, second = output.mm_items + assert len(first.identity) == len(second.identity) == 32 + assert set(first.pad_values).isdisjoint(second.pad_values) assert [item.feature.dtype for item in output.mm_items] == [ torch.uint8, torch.uint8, diff --git a/test/registered/unit/multimodal/rust/shared/test_wrap_encoded.py b/test/registered/unit/multimodal/rust/shared/test_wrap_encoded.py index 29f780301f..002ff7b063 100644 --- a/test/registered/unit/multimodal/rust/shared/test_wrap_encoded.py +++ b/test/registered/unit/multimodal/rust/shared/test_wrap_encoded.py @@ -16,6 +16,7 @@ from sglang.test.test_utils import CustomTestCase, maybe_stub_sgl_kernel maybe_stub_sgl_kernel() +from sglang.srt.managers.schedule_batch import Modality, MultimodalDataItem from sglang.srt.rust_server.multimodal import ( # noqa: E402 RustMmProcessor, RustMmSpec, @@ -107,8 +108,6 @@ class TestWrapEncoded(CustomTestCase): self.assertEqual(output.mm_items[0].feature[0, 0].item(), 99) def test_optional_pad_values_use_precomputed_hashes(self): - from sglang.srt.managers.schedule_batch import _compute_pad_value - # The whole point of worker-precomputed hashes is that the scheduler # loop never runs hash_feature — make any call a hard failure. with ( @@ -119,10 +118,11 @@ class TestWrapEncoded(CustomTestCase): ), ): output, _ = self.build() - self.assertEqual( - [item.pad_value for item in output.mm_items], - [_compute_pad_value(101), _compute_pad_value(202)], - ) + for item, item_hash in zip(output.mm_items, self.HASHES): + expected = MultimodalDataItem(modality=Modality.IMAGE, hash=item_hash) + expected.set_pad_value() + self.assertEqual(item.padding_values(), expected.padding_values()) + self.assertEqual(len(item.padding_values()), 4) class FakeShmBuffer: diff --git a/test/registered/unit/multimodal/test_kimi_image_identity.py b/test/registered/unit/multimodal/test_kimi_image_identity.py new file mode 100644 index 0000000000..6743cb1b7c --- /dev/null +++ b/test/registered/unit/multimodal/test_kimi_image_identity.py @@ -0,0 +1,326 @@ +"""Kimi image identities: full digest, config and grid folding, wide pads.""" + +import json +import unittest +from array import array +from io import BytesIO +from types import SimpleNamespace +from unittest.mock import patch + +import msgspec +import numpy as np +import torch +from PIL import Image + +from sglang.srt.environ import envs +from sglang.srt.managers.io_struct import GenerateReqInput +from sglang.srt.managers.mm_utils import ( + MultiModalityDataPaddingPatternMultimodalTokens, + MultiModalityDataPaddingPatternTokenPairs, + embed_mm_inputs, + get_new_expanded_mm_items, +) +from sglang.srt.managers.schedule_batch import ( + MM_PAD_SHIFT_VALUE, + Modality, + MultimodalDataItem, + MultimodalInputFormat, + MultimodalInputs, +) +from sglang.srt.multimodal.cache import snapshot_media +from sglang.srt.multimodal.processors.base_processor import BaseMultimodalProcessor +from sglang.srt.multimodal.processors.kimi_common import ( + KimiGridMMDataMixin, + KimiLoadedImage, + kimi_image_identity, +) +from sglang.test.ci.ci_register import register_cpu_ci +from sglang.test.test_utils import CustomTestCase + +register_cpu_ci(est_time=15, suite="base-a-test-cpu") + +# Two single-element features whose legacy 30-bit pad sentinels collide. +_LEGACY_COLLIDING_FEATURES = (23990, 59582) + + +class _IdentityOnly(KimiGridMMDataMixin): + def __init__(self, preprocess_config): + self.processor_fingerprint = json.dumps(preprocess_config, sort_keys=True) + self.hf_config = None + + +def _image(seed: int) -> Image.Image: + rng = np.random.default_rng(seed) + return Image.fromarray(rng.integers(0, 256, (28, 28, 3), dtype=np.uint8)) + + +def _image_item(grid, feature=None, fmt=MultimodalInputFormat.NORMAL): + item = MultimodalDataItem( + modality=Modality.IMAGE, + feature=feature if feature is not None else torch.zeros(1), + model_specific_data={"image_grid_thw": torch.tensor([grid])}, + ) + item.format = fmt + return item + + +class _LoadingProcessor(KimiGridMMDataMixin, BaseMultimodalProcessor): + uses_wide_image_identity = True + + +class TestKimiImageIdentity(CustomTestCase): + def test_wide_padding_reaches_embedding_scatter(self): + devices = ["cpu"] + (["cuda"] if torch.cuda.is_available() else []) + for device in devices: + with self.subTest(device=device): + item = _image_item([1, 4, 4]) + item.set_identity(kimi_image_identity("sha256:" + "12" * 32, [1, 4, 4])) + item.offsets = [(1, 6)] + item.feature = None + item.precomputed_embeddings = ( + torch.arange(18, device=device).reshape(6, 3).float() + ) + mm_inputs = MultimodalInputs(mm_items=[item], im_token_id=9) + padded = ( + MultiModalityDataPaddingPatternMultimodalTokens().pad_input_tokens( + array("q", [1, 9, 9, 9, 9, 9, 9, 2]), mm_inputs + ) + ) + embedding = torch.nn.Embedding(10, 3, device=device) + for prefix, length in ((0, 8), (3, 4)): + inputs = torch.tensor( + padded[prefix : prefix + length], device=device + ) + actual, _ = embed_mm_inputs( + [mm_inputs], + [prefix], + [length], + inputs, + embedding, + data_embedding_func_mapping={Modality.IMAGE: lambda _: None}, + ) + expected = ( + item.precomputed_embeddings + if prefix == 0 + else item.precomputed_embeddings[2:6] + ) + self.assertTrue( + torch.equal(actual[1:7] if prefix == 0 else actual, expected) + ) + + def test_loader_hashes_the_same_snapshot_it_decodes(self): + encoded = BytesIO() + _image(0).save(encoded, format="JPEG") + payload = encoded.getvalue() + loaded = _LoadingProcessor._load_single_item(payload, Modality.IMAGE) + self.assertIsInstance(loaded, KimiLoadedImage) + self.assertEqual(loaded.content_digest, snapshot_media(payload).content_digest) + item = _image_item([1, 4, 4]) + with patch( + "sglang.srt.multimodal.processors.kimi_common.snapshot_media", + side_effect=AssertionError("decoded images must not be copied for hashing"), + ): + _IdentityOnly({}).assign_kimi_image_identities([item], [loaded]) + self.assertIsNotNone(item.identity) + + def test_hf_grid_alias_has_the_same_identity(self): + processor = _IdentityOnly({}) + canonical = _image_item([1, 4, 4]) + alias = _image_item([1, 4, 4]) + alias.model_specific_data["grid_thws"] = alias.model_specific_data.pop( + "image_grid_thw" + ) + processor.assign_kimi_image_identities([canonical, alias], [_image(0)] * 2) + self.assertEqual(canonical.identity, alias.identity) + + def test_hf_bundled_grid_alias_splits_before_identity(self): + item = MultimodalDataItem( + modality=Modality.IMAGE, + feature=torch.arange(12).reshape(12, 1), + offsets=[(0, 0), (2, 3)], + model_specific_data={"grid_thws": torch.tensor([[1, 2, 2], [1, 2, 4]])}, + format=MultimodalInputFormat.PROCESSOR_OUTPUT, + ) + items = get_new_expanded_mm_items([item]) + self.assertEqual([len(entry.feature) for entry in items], [4, 8]) + _IdentityOnly({}).assign_kimi_image_identities(items, None) + self.assertNotEqual(items[0].identity, items[1].identity) + + def test_epd_embeddings_receive_content_and_grid_identity(self): + processor = _IdentityOnly({}) + processor.uses_wide_image_identity = True + processor.hf_config = SimpleNamespace( + vision_config=SimpleNamespace(merge_kernel_size=(2, 2)) + ) + identities = [] + for grid, value in (([1, 4, 4], 1), ([1, 2, 8], 1), ([1, 4, 4], 2)): + output = processor._build_kimi_mm_data_from_grids( + [9], + {Modality.IMAGE: torch.full((4, 2), value)}, + image_token_id=9, + img_grid_thw=[grid], + ) + item = output.mm_items[0] + self.assertIsNotNone(item.identity) + identities.append(item.identity) + self.assertEqual(len(set(identities)), 3) + + def test_same_pixels_different_grid_gives_different_key(self): + processor = _IdentityOnly({"in_patch_limit": 16384}) + image = _image(0) + items = [_image_item([1, 4, 4]), _image_item([1, 4, 8])] + processor.assign_kimi_image_identities(items, [image, image]) + self.assertNotEqual(items[0].identity, items[1].identity) + self.assertNotEqual(items[0].hash, items[1].hash) + self.assertTrue(set(items[0].pad_values).isdisjoint(items[1].pad_values)) + + def test_same_pixels_different_config_gives_different_key(self): + image = _image(0) + keys = [] + for limit in (16384, 4096): + item = _image_item([1, 4, 4]) + _IdentityOnly({"in_patch_limit": limit}).assign_kimi_image_identities( + [item], [image] + ) + keys.append(item) + self.assertNotEqual(keys[0].identity, keys[1].identity) + self.assertTrue(set(keys[0].pad_values).isdisjoint(keys[1].pad_values)) + + def test_same_pixels_grid_and_config_are_stable(self): + items = [_image_item([1, 4, 4]), _image_item([1, 4, 4])] + for item in items: + _IdentityOnly({"in_patch_limit": 16384}).assign_kimi_image_identities( + [item], [_image(0)] + ) + self.assertEqual(items[0].identity, items[1].identity) + self.assertEqual(items[0].pad_values, items[1].pad_values) + + def test_identity_keeps_full_digest(self): + item = _image_item([1, 4, 4]) + _IdentityOnly({}).assign_kimi_image_identities([item], [_image(0)]) + self.assertEqual(len(item.identity), 32) + self.assertEqual(item.hash, int.from_bytes(item.identity[:8], "big")) + self.assertEqual(len(item.pad_values), 4) + for value in item.pad_values: + self.assertGreaterEqual(value, MM_PAD_SHIFT_VALUE) + self.assertLess(value, 2**63) + + def test_known_legacy_collision_is_separated(self): + legacy = [] + generic = [] + for value in _LEGACY_COLLIDING_FEATURES: + item = MultimodalDataItem( + modality=Modality.IMAGE, + feature=torch.tensor([value], dtype=torch.int64), + ) + item.set_pad_value() + legacy.append(MM_PAD_SHIFT_VALUE + item.hash % (1 << 30)) + generic.append(item.pad_value) + self.assertEqual(legacy[0], legacy[1]) + self.assertNotEqual(generic[0], generic[1]) + + items = [ + _image_item( + [1, 4, 4], + feature=torch.tensor([value], dtype=torch.int64), + fmt=MultimodalInputFormat.PROCESSOR_OUTPUT, + ) + for value in _LEGACY_COLLIDING_FEATURES + ] + _IdentityOnly({}).assign_kimi_image_identities(items, None) + self.assertNotEqual(items[0].identity, items[1].identity) + self.assertTrue(set(items[0].pad_values).isdisjoint(items[1].pad_values)) + + def test_many_distinct_images_have_disjoint_pad_values(self): + seen = set() + for seed in range(256): + item = _image_item([1, 4, 4]) + _IdentityOnly({}).assign_kimi_image_identities([item], [_image(seed)]) + self.assertTrue(seen.isdisjoint(item.pad_values)) + seen.update(item.pad_values) + + def test_skip_hash_keeps_legacy_contract(self): + item = _image_item([1, 4, 4]) + with envs.SGLANG_MM_SKIP_COMPUTE_HASH.override(True): + _IdentityOnly({}).assign_kimi_image_identities([item], [_image(0)]) + self.assertIsNone(item.identity) + self.assertIsNone(item.pad_values) + + def test_set_hash_clears_wide_identity(self): + item = _image_item([1, 4, 4]) + item.set_identity(kimi_image_identity("sha256:" + "ab" * 32, [1, 4, 4])) + item.set_hash(7) + self.assertIsNone(item.identity) + expected = MultimodalDataItem(modality=Modality.IMAGE, hash=7) + expected.set_pad_value() + self.assertEqual(item.padding_values(), expected.padding_values()) + self.assertEqual(len(item.padding_values()), 4) + + def test_rejects_short_identity(self): + with self.assertRaises(ValueError): + _image_item([1, 4, 4]).set_identity(b"\x00" * 8) + + def test_wide_pads_fill_spans_and_placeholders(self): + item = _image_item([1, 4, 4]) + item.set_identity(kimi_image_identity("sha256:" + "cd" * 32, [1, 4, 4])) + item.offsets = [(1, 6)] + mm_inputs = MultimodalInputs(mm_items=[item], im_token_id=9) + padded = MultiModalityDataPaddingPatternMultimodalTokens().pad_input_tokens( + array("q", [0, 9, 9, 9, 9, 9, 9, 2]), mm_inputs + ) + values = item.pad_values + self.assertEqual(list(padded[1:7]), [values[i % 4] for i in range(6)]) + self.assertEqual(padded[0], 0) + self.assertEqual(padded[7], 2) + self.assertEqual(item.padding_sequence(6), list(padded[1:7])) + + def test_wide_pads_fill_token_pairs_without_mutating_input(self): + item = _image_item([1, 4, 4]) + item.set_identity(kimi_image_identity("sha256:" + "cd" * 32, [1, 4, 4])) + mm_inputs = MultimodalInputs(mm_items=[item]) + tokens = array("q", [0, 10, 9, 9, 9, 9, 9, 9, 11, 2]) + original = tokens[:] + padded = MultiModalityDataPaddingPatternTokenPairs([(10, 11)]).pad_input_tokens( + tokens, mm_inputs + ) + self.assertEqual(list(padded[2:8]), item.padding_sequence(6)) + self.assertEqual(list(padded[:2]), [0, 10]) + self.assertEqual(list(padded[8:]), [11, 2]) + self.assertEqual(mm_inputs.data_offsets, [1]) + self.assertEqual(tokens, original) + + def test_wide_identity_survives_msgspec_roundtrip(self): + item = MultimodalDataItem(modality=Modality.IMAGE) + item.set_identity(kimi_image_identity("sha256:" + "ef" * 32, [1, 2, 2])) + decoded = msgspec.msgpack.decode( + msgspec.msgpack.encode(item), type=MultimodalDataItem + ) + self.assertEqual(decoded.identity, item.identity) + self.assertEqual(decoded.pad_values, item.pad_values) + self.assertEqual(decoded.pad_value, item.pad_value) + self.assertEqual(decoded.hash, item.hash) + + +class TestKimiCallerIdentityRejection(CustomTestCase): + def test_rejects_mm_hashes(self): + request = GenerateReqInput(text="x", image_data=["image"], mm_hashes=["07"]) + with self.assertRaisesRegex(ValueError, "mm_hashes"): + KimiGridMMDataMixin.reject_caller_image_identity(["image"], request) + + def test_rejects_identity_fields_in_image_dicts(self): + request = GenerateReqInput(text="x", image_data=["image"]) + for key in ("hash", "pad_value", "pad_values", "identity"): + with self.subTest(key=key), self.assertRaisesRegex(ValueError, key): + KimiGridMMDataMixin.reject_caller_image_identity( + [{"format": "processor_output", key: 1}], request + ) + + def test_accepts_plain_images(self): + request = GenerateReqInput(text="x", image_data=["image"]) + KimiGridMMDataMixin.reject_caller_image_identity( + ["image", {"format": "processor_output"}], request + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/unit/spec/test_ngram_worker_prompt.py b/test/registered/unit/spec/test_ngram_worker_prompt.py new file mode 100644 index 0000000000..147b06f3e7 --- /dev/null +++ b/test/registered/unit/spec/test_ngram_worker_prompt.py @@ -0,0 +1,87 @@ +import unittest +from array import array +from types import SimpleNamespace + +import numpy as np +import torch + +from sglang.test.ci.ci_register import register_cpu_ci +from sglang.test.test_utils import CustomTestCase, maybe_stub_sgl_kernel + +register_cpu_ci(est_time=20, suite="base-a-test-cpu") +maybe_stub_sgl_kernel() + +from sglang.srt.managers.schedule_batch import Req +from sglang.srt.sampling.sampling_params import SamplingParams +from sglang.srt.speculative.cpp_ngram.ngram_corpus import NgramCorpus +from sglang.srt.speculative.ngram_worker import NGRAMWorker + + +class TestNgramWorkerPrompt(CustomTestCase): + def test_multimodal_padding_stays_out_of_corpus(self): + for overlap, grammar in [(False, False), (True, False), (True, True)]: + for multimodal in [False, True]: + with self.subTest( + overlap=overlap, grammar=grammar, multimodal=multimodal + ): + worker = object.__new__(NGRAMWorker) + worker.enable_overlap = overlap + worker.max_trie_depth = 12 + worker.draft_token_num = 4 + worker.ngram_corpus = NgramCorpus( + max_trie_depth=12, + min_bfs_breadth=1, + max_bfs_breadth=1, + draft_token_num=4, + capacity=1000, + ) + req = Req("prompt", "", array("q", [1, 2, 3]), SamplingParams()) + if multimodal: + # scheduler replaces cache IDs while preserving the original prompt + req.origin_input_ids = array("q", [1] + [1 << 60] * 16 + [2, 3]) + req.output_ids = array("q", [4]) + splice = overlap and not grammar + batch = SimpleNamespace( + reqs=[req], + grammar_needs_sync=lambda: grammar, + spec_info=SimpleNamespace( + accept_tokens=torch.tensor([5, 0, 0, 0]), + accept_lens=torch.tensor([1]), + ), + ) + sequence = [1, 2, 3, 4, 5, 6, 7, 8] + worker.ngram_corpus.batch_put([sequence]) + worker.ngram_corpus.synchronize() + reference = NgramCorpus( + max_trie_depth=12, + min_bfs_breadth=1, + max_bfs_breadth=1, + draft_token_num=4, + capacity=1000, + ) + reference.batch_put([sequence]) + reference.synchronize() + padded = req.origin_input_ids[:] + for step in range(2): + tokens = ( + [1, 2, 3] + list(req.output_ids) + ([5] if splice else []) + ) + expected = reference.batch_get( + [req.rid], [tokens], [len(tokens)] + ) + actual = worker._prepare_draft_tokens(batch) + np.testing.assert_array_equal(actual[0], expected[0]) + np.testing.assert_array_equal(actual[1], expected[1]) + worker._update_ngram_corpus(batch) + worker.ngram_corpus.synchronize() + inserted = worker.ngram_corpus.batch_get( + [f"inserted-{step}"], [[1, 2]], [2] + )[0] + self.assertIn(3, inserted) + req.output_ids.append(6) + self.assertEqual(req.origin_input_ids, padded) + self.assertEqual(list(req.origin_input_ids_unpadded), [1, 2, 3]) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/vlm/test_vlm_input_format.py b/test/registered/vlm/test_vlm_input_format.py index 6b5a94b6c0..3b3f245b3a 100644 --- a/test/registered/vlm/test_vlm_input_format.py +++ b/test/registered/vlm/test_vlm_input_format.py @@ -222,6 +222,49 @@ class TestQwenVLUnderstandsImage(VLMInputTestBase, CustomTestCase): model_path = "Qwen/Qwen2.5-VL-3B-Instruct" chat_template = "qwen2-vl" + def test_colliding_low_hash_bits_do_not_reuse_image_kv(self): + prompt = self.processor.apply_chat_template( + [ + { + "role": "user", + "content": [ + {"type": "image"}, + {"type": "text", "text": "Name the image's color in one word."}, + ], + } + ], + tokenize=False, + add_generation_prompt=True, + ) + red = Image.new("RGB", (224, 224), "red") + blue = Image.new("RGB", (224, 224), "blue") + params = dict(temperature=0.0, max_new_tokens=16) + # these distinct caller hashes collapse to the same legacy 30-bit pad + self.engine.flush_cache() + cold_blue = self.engine.generate( + prompt=prompt, + image_data=[blue], + mm_hashes=["40000007"], + sampling_params=params, + ) + self.engine.flush_cache() + red_output = self.engine.generate( + prompt=prompt, image_data=[red], mm_hashes=["7"], sampling_params=params + ) + warm_blue = self.engine.generate( + prompt=prompt, + image_data=[blue], + mm_hashes=["40000007"], + sampling_params=params, + ) + self.assertIn("red", red_output["text"].lower()) + self.assertIn("blue", cold_blue["text"].lower()) + self.assertEqual(warm_blue["text"], cold_blue["text"]) + self.assertLess( + warm_blue["meta_info"]["cached_tokens"], + warm_blue["meta_info"]["prompt_tokens"] - 1, + ) + @classmethod def _init_visual(cls): model = Qwen2_5_VLForConditionalGeneration.from_pretrained(