From 010776e94f31a94106fdeeb723fcdd951cb6c4ee Mon Sep 17 00:00:00 2001 From: Kan Wu Date: Wed, 14 Oct 2026 07:56:28 -0700 Subject: [PATCH] [KV events] Publish opaque LoRA block IDs and adapter names (#42817) Co-authored-by: Kan Wu Co-authored-by: Claude Opus 5.5 Co-authored-by: Shangming Cai --- python/sglang/srt/disaggregation/kv_events.py | 4 +- python/sglang/srt/managers/io_struct.py | 4 + python/sglang/srt/managers/scheduler.py | 17 +- .../sglang/srt/managers/tokenizer_manager.py | 2 + .../sglang/srt/mem_cache/cache_init_params.py | 9 +- python/sglang/srt/mem_cache/events.py | 64 ++++++- .../sglang/srt/mem_cache/kv_cache_builder.py | 4 +- python/sglang/srt/mem_cache/radix_cache.py | 2 +- .../srt/mem_cache/rust_tree_core/adapter.py | 68 +++++-- rust/sglang-radix-tree/src/python_bindings.rs | 10 +- .../src/tests/unified_tree_core.rs | 48 ++++- .../src/unified_tree_core.rs | 22 ++- .../unit/disaggregation/test_kv_events.py | 5 + .../dllm/test_gemma4_uniform_lifecycle.py | 1 + ...test_scheduler_sampling_mask_validation.py | 1 + .../unit/mem_cache/test_radix_cache_unit.py | 47 +++++ .../test_unified_radix_cache_unittest.py | 177 ++++++++++++++++++ .../multimodal/test_gpu_feature_transport.py | 2 + 18 files changed, 446 insertions(+), 41 deletions(-) diff --git a/python/sglang/srt/disaggregation/kv_events.py b/python/sglang/srt/disaggregation/kv_events.py index 0597c72dfd..84d3855034 100644 --- a/python/sglang/srt/disaggregation/kv_events.py +++ b/python/sglang/srt/disaggregation/kv_events.py @@ -290,8 +290,8 @@ class BlockStored(KVCacheEvent): # Session that triggered this store. Attribution only: the blocks may be # shared with other sessions, and the hash does not depend on it. session_id: Optional[str] = None - # The LoRA adapter name of the request that stored these blocks. - # Set only in the dynamo format. + # Public adapter name for router lookup. Published hashes are opaque IDs + # scoped to the internal cache namespace, including the adapter load. lora_name: Optional[str] = None diff --git a/python/sglang/srt/managers/io_struct.py b/python/sglang/srt/managers/io_struct.py index e32fc01140..d3ee899270 100644 --- a/python/sglang/srt/managers/io_struct.py +++ b/python/sglang/srt/managers/io_struct.py @@ -1173,6 +1173,8 @@ class TokenizedGenerateReqInput(BaseReq, kw_only=True): # Shape of output_token_sampling_logprobs for each output token. This is a # defaulted tail field so older IPC senders decode as selected mode. sampling_logprobs_mode: SamplingLogprobsMode = "selected" + # LoRA adapter name; KV events namespace this request's blocks by it. + lora_name: Optional[str] = None def wrap_pickle_fields(self): self.time_stats = wrap_as_pickle(self.time_stats) @@ -1537,6 +1539,8 @@ class TokenizedEmbeddingReqInput(BaseReq, kw_only=True): # For observability # Pickled Optional[Union[APIServerReqTimeStats, DPControllerReqTimeStats]] time_stats: Optional[PickleWrapper] = None + # LoRA adapter name; KV events namespace this request's blocks by it. + lora_name: Optional[str] = None def wrap_pickle_fields(self): self.time_stats = wrap_as_pickle(self.time_stats) diff --git a/python/sglang/srt/managers/scheduler.py b/python/sglang/srt/managers/scheduler.py index 79efa0bacc..509ee26322 100644 --- a/python/sglang/srt/managers/scheduler.py +++ b/python/sglang/srt/managers/scheduler.py @@ -298,7 +298,7 @@ from sglang.srt.mem_cache.common import ( discard_kv_cache_backup, release_kv_cache, ) -from sglang.srt.mem_cache.events import LoRANameTable +from sglang.srt.mem_cache.events import KvEventLoraNames from sglang.srt.model_executor.forward_batch_info import PPProxyTensors from sglang.srt.model_executor.runner_utils.pool import prewarm_graph_pool_borrow from sglang.srt.model_loader.utils import get_resolved_model_impl @@ -2451,7 +2451,9 @@ class Scheduler( def init_kv_event_lora_names(self) -> None: """Init the table that maps lora_id to adapter name for KV events.""" - self.kv_event_lora_names = LoRANameTable.from_lora_refs(get_lora().lora_paths) + self.kv_event_lora_names = KvEventLoraNames.from_lora_refs( + get_lora().lora_paths + ) def init_kv_events_publisher(self) -> None: self.kv_events_publisher = SchedulerKvEventsPublisher( @@ -2778,6 +2780,15 @@ class Scheduler( mm.mrope_positions = mrope_positions mm.mrope_position_delta = mrope_position_delta + def _record_kv_event_lora_name(self, req: Req, lora_name: Optional[str]) -> None: + # Keyed by the final extra_key, so it must run after elastic namespacing. + # Only a cache whose events are taken prunes the table. + if ( + self.kv_events_publisher.enable_kv_cache_events + and self.tree_cache.supports_prefix_sharing() + ): + self.kv_event_lora_names.register(req, lora_name) + def _maybe_namespace_elastic_radix_cache(self, req: Req) -> None: if ( get_exec().moe.elastic_ep_backend is None @@ -2990,6 +3001,7 @@ class Scheduler( if recv_req.pp_prefetch_ticketed is True: self.tree_cache.bind_prefetch_ticket(req.rid) self._maybe_namespace_elastic_radix_cache(req) + self._record_kv_event_lora_name(req, recv_req.lora_name) if mm_input_error is not None: req.set_finish_with_abort( @@ -3566,6 +3578,7 @@ class Scheduler( ) req.tokenizer = self.tokenizer self._maybe_namespace_elastic_radix_cache(req) + self._record_kv_event_lora_name(req, recv_req.lora_name) if mm_input_error is not None: req.set_finish_with_abort( diff --git a/python/sglang/srt/managers/tokenizer_manager.py b/python/sglang/srt/managers/tokenizer_manager.py index 3dd350f411..7a5eef52f1 100644 --- a/python/sglang/srt/managers/tokenizer_manager.py +++ b/python/sglang/srt/managers/tokenizer_manager.py @@ -1602,6 +1602,7 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin): bootstrap_port=obj.bootstrap_port, bootstrap_room=bootstrap_room, lora_id=obj.lora_id, + lora_name=obj.lora_path, input_embeds=input_embeds, positional_embed_overrides=obj.positional_embed_overrides, session_id=obj.session_id, @@ -1649,6 +1650,7 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin): priority=obj.priority, dimensions=obj.dimensions, lora_id=obj.lora_id, + lora_name=obj.lora_path, http_worker_ipc=obj.http_worker_ipc, return_pooled_hidden_states=obj.return_pooled_hidden_states, multi_item_delimiter_indices=obj.multi_item_delimiter_indices, diff --git a/python/sglang/srt/mem_cache/cache_init_params.py b/python/sglang/srt/mem_cache/cache_init_params.py index 9794b59cc2..620847bc8f 100644 --- a/python/sglang/srt/mem_cache/cache_init_params.py +++ b/python/sglang/srt/mem_cache/cache_init_params.py @@ -5,9 +5,10 @@ from typing import TYPE_CHECKING, Any, Optional import torch +from sglang.srt.mem_cache.events import KvEventLoraNames + if TYPE_CHECKING: from sglang.srt.mem_cache.allocator import BaseTokenToKVPoolAllocator - from sglang.srt.mem_cache.events import LoRANameTable from sglang.srt.mem_cache.memory_pool import ReqToTokenPool from sglang.srt.mem_cache.unified_cache.components import ComponentType from sglang.srt.mem_cache.unified_cache.components.base import ( @@ -35,8 +36,10 @@ class CacheInitParams: enable_metrics: bool = False enable_kv_cache_events: bool = False dynamo_kv_event_format: bool = False - # The scheduler holds the same table and adds each adapter when it loads. - kv_event_lora_names: Optional[LoRANameTable] = None + # LoRA names for KV events; the scheduler registers requests in it. + kv_event_lora_names: KvEventLoraNames = dataclasses.field( + default_factory=KvEventLoraNames + ) enable_session_radix_cache: bool = False enable_mamba_extra_buffer: bool = False diff --git a/python/sglang/srt/mem_cache/events.py b/python/sglang/srt/mem_cache/events.py index cf7d8c5d4b..861d8589d2 100644 --- a/python/sglang/srt/mem_cache/events.py +++ b/python/sglang/srt/mem_cache/events.py @@ -65,12 +65,57 @@ class LoRANameTable: def register(self, *, lora_id: str, lora_name: str) -> None: self._names[lora_id] = lora_name + def get(self, extra_key: Optional[str]) -> Optional[str]: + return self.resolve(extra_key) + def resolve(self, extra_key: Optional[str]) -> Optional[str]: if not extra_key: return None return self._names.get(extra_key[-_LORA_ID_LEN:]) +class KvEventLoraNames(LoRANameTable): + """Public names of selected LoRA namespaces, kept for the publisher lifetime. + + Write-back can publish a host store after the last GPU removal and after + its request dies, so request or published-block counts cannot bound this. + """ + + def __init__(self): + # Adapter-id registrations keep the existing Dynamo load-time lookup. + self._adapter_names: dict[str, str] = {} + self._names: dict[str, str] = {} + + def register( + self, + req: Any = None, + lora_name: Optional[str] = None, + *, + lora_id: Optional[str] = None, + ) -> None: + """Name ``req``'s namespace; ``req`` needs ``extra_key`` and ``lora_id``.""" + if lora_id is not None: + self._adapter_names[lora_id] = lora_name + return + if req.lora_id is None: + return + # Older IPC senders omit the optional name; admission still resolves an ID. + lora_name = lora_name or self._adapter_names.get(req.lora_id) + if not lora_name: + return + self._names[req.extra_key] = lora_name + + def get(self, extra_key: Optional[str]) -> Optional[str]: + return self._names.get(extra_key) + + def resolve(self, extra_key: Optional[str]) -> Optional[str]: + if not extra_key: + return None + return self._names.get(extra_key) or self._adapter_names.get( + extra_key[-_LORA_ID_LEN:] + ) + + class KVCacheEventRecorder: """Collects KV placement events for one cache. @@ -84,12 +129,12 @@ class KVCacheEventRecorder: enabled: bool, page_size: int, dynamo_format: bool = False, - lora_names: Optional[LoRANameTable] = None, + lora_names: Optional[KvEventLoraNames] = None, ): self.enabled = enabled self.page_size = page_size self.dynamo_format = dynamo_format - self.lora_names = lora_names if lora_names is not None else LoRANameTable() + self.lora_names = lora_names if lora_names is not None else KvEventLoraNames() self._queue: list = [] def enqueue(self, event) -> None: @@ -114,6 +159,7 @@ class KVCacheEventRecorder: and tail.block_size == event.block_size and tail.cache_salt == event.cache_salt and tail.session_id == event.session_id + and tail.lora_name == event.lora_name and tail.block_hashes and event.parent_block_hash == tail.block_hashes[-1] ): @@ -151,9 +197,15 @@ class KVCacheEventRecorder: return hash_str_to_int64(parent_hash_values[-1]) def _namespace_seed(self, node: Any) -> Optional[bytes]: - # The default format publishes the hashes unchanged. if not self.dynamo_format: - return None + return kv_event_namespace_seed( + extra_key=( + node.key.extra_key + if self.lora_names.get(node.key.extra_key) is not None + else None + ), + cache_salt=None, + ) return kv_event_namespace_seed( extra_key=node.key.extra_key, cache_salt=node.key.cache_salt ) @@ -177,7 +229,9 @@ class KVCacheEventRecorder: parent_block_hash, namespace_seed=namespace_seed ) lora_name = ( - self.lora_names.resolve(node.key.extra_key) if self.dynamo_format else None + self.lora_names.resolve(node.key.extra_key) + if self.dynamo_format + else self.lora_names.get(node.key.extra_key) ) page_index = 0 diff --git a/python/sglang/srt/mem_cache/kv_cache_builder.py b/python/sglang/srt/mem_cache/kv_cache_builder.py index 488fa72667..290bdfa01b 100644 --- a/python/sglang/srt/mem_cache/kv_cache_builder.py +++ b/python/sglang/srt/mem_cache/kv_cache_builder.py @@ -40,6 +40,7 @@ from sglang.srt.managers.mm_schedule import init_mm_embedding_cache from sglang.srt.mem_cache.base_swa_memory_pool import BaseSWAKVPool from sglang.srt.mem_cache.cache_init_params import CacheInitParams from sglang.srt.mem_cache.deepseek_v4_memory_pool import DeepSeekV4TokenToKVPool +from sglang.srt.mem_cache.events import KvEventLoraNames from sglang.srt.mem_cache.hicache_auto_size import auto_size_hicache from sglang.srt.mem_cache.memory_pool import HybridLinearKVPool, MHATokenToKVPool from sglang.srt.mem_cache.pool_host.base import _WRITE_BACK_STAGING_PAGE_CHUNK @@ -61,7 +62,6 @@ from sglang.srt.utils import ceil_align, is_hip if TYPE_CHECKING: from sglang.srt.configs.model_config import ModelConfig from sglang.srt.managers.tp_worker import BaseTpWorker - from sglang.srt.mem_cache.events import LoRANameTable from sglang.srt.server_args import ServerArgs from sglang.srt.speculative.base_spec_worker import HiCacheDraftPlan from sglang.srt.speculative.spec_info import SpeculativeAlgorithm @@ -257,7 +257,7 @@ def build_kv_cache( enable_hierarchical_cache: bool, hicache_draft_plan: Optional[HiCacheDraftPlan] = None, dynamo_kv_event_format: bool = False, - kv_event_lora_names: Optional[LoRANameTable] = None, + kv_event_lora_names: Optional[KvEventLoraNames] = None, ) -> KVCacheBuildResult: parallel = get_parallel() sliding_window_size: Optional[int] = None diff --git a/python/sglang/srt/mem_cache/radix_cache.py b/python/sglang/srt/mem_cache/radix_cache.py index b034e4e9c2..b447f015cb 100644 --- a/python/sglang/srt/mem_cache/radix_cache.py +++ b/python/sglang/srt/mem_cache/radix_cache.py @@ -71,7 +71,7 @@ class RadixKey: ): # token ids sequence (raw ints in both modes) self.token_ids = token_ids - # Namespaces the tree and storage; omitted from KV events. + # Namespaces the tree and storage; KV events see only its LoRA name. self.extra_key = extra_key # Namespaces the tree, storage and KV events. self.cache_salt = cache_salt or None diff --git a/python/sglang/srt/mem_cache/rust_tree_core/adapter.py b/python/sglang/srt/mem_cache/rust_tree_core/adapter.py index 6fb481ccd5..3b1040b208 100644 --- a/python/sglang/srt/mem_cache/rust_tree_core/adapter.py +++ b/python/sglang/srt/mem_cache/rust_tree_core/adapter.py @@ -14,6 +14,8 @@ from sglang.srt.disaggregation.kv_events import ( BlockRemoved, BlockStored, StorageMedium, + kv_event_namespace_seed, + namespaced_block_hash, ) from sglang.srt.environ import envs from sglang.srt.mem_cache.allocator.swa import is_swa_req_ring @@ -26,7 +28,7 @@ from sglang.srt.mem_cache.base_prefix_cache import ( MatchPrefixParams, MatchResult, ) -from sglang.srt.mem_cache.events import LoRANameTable +from sglang.srt.mem_cache.events import KvEventLoraNames from sglang.srt.mem_cache.hicache_storage import PoolHitPolicy, PoolName, PoolTransfer from sglang.srt.mem_cache.radix_cache import RadixKey from sglang.srt.mem_cache.rust_tree_core.extension import bindings @@ -60,7 +62,9 @@ from sglang.srt.mem_cache.unified_cache.unified_tree_core_interface import ( TreePathNodeInfo, UnifiedTreeCoreInterface, ) -from sglang.srt.mem_cache.utils import get_eviction_strategy +from sglang.srt.mem_cache.utils import ( + get_eviction_strategy, +) from sglang.srt.runtime_context import get_exec, mamba_cache_chunk_size from sglang.srt.utils import assert_int64_array @@ -97,23 +101,54 @@ def _radix_key_buffer(key: RadixKey) -> array: return token_ids -def _kv_event_from_tagged(event: tuple, lora_names: LoRANameTable): - """Build the Python KV cache event for one of the binding's tagged tuples.""" +def _kv_event_from_tagged( + event: tuple, lora_names: KvEventLoraNames, dynamo_format: bool = False +): + """Build the Python KV cache event for one of the binding's tagged tuples. + + Rust hashes are LoRA-agnostic; the trailing extra_key selects the LoRA + namespace, applied here as in KVCacheEventRecorder. + """ tag = event[0] if tag == "block_stored": + lora_name = ( + lora_names.resolve(event[8]) if dynamo_format else lora_names.get(event[8]) + ) + seed = ( + kv_event_namespace_seed(extra_key=event[8], cache_salt=None) + if not dynamo_format and lora_name is not None + else None + ) + parent = event[2] return BlockStored( - block_hashes=event[1], - parent_block_hash=event[2], + block_hashes=[ + namespaced_block_hash(h, namespace_seed=seed) for h in event[1] + ], + parent_block_hash=( + None + if parent is None + else namespaced_block_hash(parent, namespace_seed=seed) + ), token_ids=event[3], block_size=event[4], lora_id=None, medium=StorageMedium(event[5]), cache_salt=event[6], session_id=event[7], - lora_name=lora_names.resolve(event[8]), + lora_name=lora_name, ) if tag == "block_removed": - return BlockRemoved(block_hashes=event[1], medium=StorageMedium(event[2])) + seed = ( + kv_event_namespace_seed(extra_key=event[3], cache_salt=None) + if not dynamo_format and lora_names.get(event[3]) is not None + else None + ) + return BlockRemoved( + block_hashes=[ + namespaced_block_hash(h, namespace_seed=seed) for h in event[1] + ], + medium=StorageMedium(event[2]), + ) if tag == "all_blocks_cleared": return AllBlocksCleared() raise ValueError(f"unknown kv event tag: {tag}") @@ -305,20 +340,26 @@ class _RustKVCacheEventRecorder: """Expose the Rust event queue through the Python recorder interface.""" def __init__( - self, binding, enabled: bool, lora_names: Optional[LoRANameTable] = None + self, + binding, + enabled: bool, + lora_names: KvEventLoraNames, + dynamo_format: bool = False, ): self._binding = binding self.enabled = enabled - self.lora_names = lora_names if lora_names is not None else LoRANameTable() + self._lora_names = lora_names + self._dynamo_format = dynamo_format def record_all_cleared(self) -> None: self._binding.record_all_cleared_event() def take(self) -> list: - return [ - _kv_event_from_tagged(event, self.lora_names) + events = [ + _kv_event_from_tagged(event, self._lora_names, self._dynamo_format) for event in self._binding.take_events() ] + return events class RustUnifiedTreeCore(UnifiedTreeCoreInterface): @@ -445,8 +486,9 @@ class RustUnifiedTreeCore(UnifiedTreeCoreInterface): ) self.kv_events = _RustKVCacheEventRecorder( self._binding, - params.enable_kv_cache_events, + enabled=params.enable_kv_cache_events, lora_names=params.kv_event_lora_names, + dynamo_format=params.dynamo_kv_event_format, ) # The default-root empty result, prebuilt once from the binding. self._empty_match_result = _match_result_from_binding( diff --git a/rust/sglang-radix-tree/src/python_bindings.rs b/rust/sglang-radix-tree/src/python_bindings.rs index 168c98120c..20a95366b1 100644 --- a/rust/sglang-radix-tree/src/python_bindings.rs +++ b/rust/sglang-radix-tree/src/python_bindings.rs @@ -2202,9 +2202,15 @@ impl TreeCoreBinding { KvCacheEvent::BlockRemoved { block_hashes, medium, + extra_key, } => { - let item: Py = - ("block_removed", block_hashes, medium.as_str()).into_py_any(py)?; + let item: Py = ( + "block_removed", + block_hashes, + medium.as_str(), + extra_key.map(|extra_key| extra_key.to_string()), + ) + .into_py_any(py)?; list.append(item)?; } KvCacheEvent::AllBlocksCleared => { diff --git a/rust/sglang-radix-tree/src/tests/unified_tree_core.rs b/rust/sglang-radix-tree/src/tests/unified_tree_core.rs index 1027a43025..64dfb871a9 100644 --- a/rust/sglang-radix-tree/src/tests/unified_tree_core.rs +++ b/rust/sglang-radix-tree/src/tests/unified_tree_core.rs @@ -3382,7 +3382,8 @@ fn namespaced_event_hashes_are_sparse_and_removed_with_the_node() { #[test] fn extra_key_nodes_publish_token_only_event_hashes() { - // Events omit extra_key; storage includes it. + // Event hashes omit extra_key; storage includes it. The event carries it + // so Python can apply the LoRA namespace. let mut tc = events_core(2); let key = vec![1, 2, 7, 8]; tc.insert(&insert_params_in_namespace( @@ -3405,7 +3406,7 @@ fn extra_key_nodes_publish_token_only_event_hashes() { medium: StorageMedium::Gpu, cache_salt: None, session_id: None, - extra_key: None, + extra_key: Some(Arc::from("lora-a")), }] ); @@ -3541,11 +3542,13 @@ fn event_coalescing_respects_store_remove_and_clear_boundaries() { tc.enqueue_kv_event_(KvCacheEvent::BlockRemoved { block_hashes: vec![1], medium: StorageMedium::Gpu, + extra_key: None, }); assert_eq!(tc.kv_event_queue.len(), 5); tc.enqueue_kv_event_(KvCacheEvent::BlockRemoved { block_hashes: vec![2, 3], medium: StorageMedium::Gpu, + extra_key: None, }); assert_eq!(tc.kv_event_queue.len(), 5); assert!(matches!( @@ -3556,6 +3559,7 @@ fn event_coalescing_respects_store_remove_and_clear_boundaries() { tc.enqueue_kv_event_(KvCacheEvent::BlockRemoved { block_hashes: vec![4], medium: StorageMedium::Cpu, + extra_key: None, }); assert_eq!(tc.kv_event_queue.len(), 6); tc.record_all_cleared_event(); @@ -3563,6 +3567,7 @@ fn event_coalescing_respects_store_remove_and_clear_boundaries() { tc.enqueue_kv_event_(KvCacheEvent::BlockRemoved { block_hashes: vec![5], medium: StorageMedium::Cpu, + extra_key: None, }); assert_eq!(tc.kv_event_queue.len(), 8); assert!(matches!( @@ -3592,6 +3597,41 @@ fn event_coalescing_respects_store_remove_and_clear_boundaries() { extra_key: None, }); assert_eq!(tc.kv_event_queue.len(), 2); + + // Different extra_keys map to different LoRA namespaces in Python. + tc.kv_event_queue.clear(); + tc.enqueue_kv_event_(KvCacheEvent::BlockStored { + block_hashes: vec![1], + parent_block_hash: None, + token_ids: vec![10, 11], + block_size: 2, + medium: StorageMedium::Gpu, + cache_salt: None, + session_id: None, + extra_key: None, + }); + tc.enqueue_kv_event_(KvCacheEvent::BlockStored { + block_hashes: vec![2], + parent_block_hash: Some(1), + token_ids: vec![12, 13], + block_size: 2, + medium: StorageMedium::Gpu, + cache_salt: None, + session_id: None, + extra_key: Some(Arc::from("lora-a")), + }); + assert_eq!(tc.kv_event_queue.len(), 2); + tc.enqueue_kv_event_(KvCacheEvent::BlockRemoved { + block_hashes: vec![1], + medium: StorageMedium::Gpu, + extra_key: None, + }); + tc.enqueue_kv_event_(KvCacheEvent::BlockRemoved { + block_hashes: vec![2], + medium: StorageMedium::Gpu, + extra_key: Some(Arc::from("lora-a")), + }); + assert_eq!(tc.kv_event_queue.len(), 4); } #[test] @@ -3621,6 +3661,7 @@ fn eviction_emits_block_removed_with_all_page_hashes() { .map(|h| crate::node::hash_str_to_int64(h)) .collect(), medium: StorageMedium::Gpu, + extra_key: None, }] ); } @@ -3831,6 +3872,7 @@ fn host_eviction_emits_a_cpu_block_removed() { vec![KvCacheEvent::BlockRemoved { block_hashes: vec![crate::node::hash_str_to_int64(&hashes[0])], medium: StorageMedium::Cpu, + extra_key: None, }] ); } @@ -3931,6 +3973,7 @@ fn drop_subtree_emits_removals_for_host_descendants_then_the_leaf() { .map(|h| crate::node::hash_str_to_int64(h)) .collect(), medium: StorageMedium::Cpu, + extra_key: None, }, KvCacheEvent::BlockRemoved { block_hashes: leaf_hashes @@ -3938,6 +3981,7 @@ fn drop_subtree_emits_removals_for_host_descendants_then_the_leaf() { .map(|h| crate::node::hash_str_to_int64(h)) .collect(), medium: StorageMedium::Gpu, + extra_key: None, }, ] ); diff --git a/rust/sglang-radix-tree/src/unified_tree_core.rs b/rust/sglang-radix-tree/src/unified_tree_core.rs index 3c9df474d4..6ff9e5d523 100644 --- a/rust/sglang-radix-tree/src/unified_tree_core.rs +++ b/rust/sglang-radix-tree/src/unified_tree_core.rs @@ -3276,12 +3276,16 @@ impl UnifiedTreeCore { Some(KvCacheEvent::BlockRemoved { block_hashes: tail_hashes, medium: tail_medium, + extra_key: tail_extra_key, }), KvCacheEvent::BlockRemoved { mut block_hashes, medium, + extra_key, }, - ) if *tail_medium == medium => tail_hashes.append(&mut block_hashes), + ) if *tail_medium == medium && *tail_extra_key == extra_key => { + tail_hashes.append(&mut block_hashes) + } ( Some(KvCacheEvent::BlockStored { block_hashes: tail_hashes, @@ -3290,6 +3294,7 @@ impl UnifiedTreeCore { medium: tail_medium, cache_salt: tail_cache_salt, session_id: tail_session_id, + extra_key: tail_extra_key, .. }), KvCacheEvent::BlockStored { @@ -3300,12 +3305,13 @@ impl UnifiedTreeCore { medium, cache_salt, session_id, - .. + extra_key, }, ) if *tail_medium == medium && *tail_block_size == block_size && *tail_cache_salt == cache_salt && *tail_session_id == session_id + && *tail_extra_key == extra_key && !tail_hashes.is_empty() && parent_block_hash == tail_hashes.last().copied() => { @@ -3386,11 +3392,7 @@ impl UnifiedTreeCore { self.arena.node_mut(node_id).hash_value = Some(hash_values); } let cache_salt = self.arena.node(node_id).namespace.cache_salt_arc(); - // Python reads the LoRA adapter name from extra_key in the dynamo format. - let extra_key = self - .dynamo_kv_event_format - .then(|| self.arena.node(node_id).namespace.extra_key_arc()) - .flatten(); + let extra_key = self.arena.node(node_id).namespace.extra_key_arc(); let session_id: Option> = session_id.map(Arc::from); let namespaced = self.arena.node(node_id).namespace != KeyNamespace::default(); if namespaced { @@ -3491,9 +3493,11 @@ impl UnifiedTreeCore { .map(|hash| namespaced_block_hash(namespace_seed.as_ref(), hash)) .collect(); if !block_hashes.is_empty() { + let extra_key = node.namespace.extra_key_arc(); self.enqueue_kv_event_(KvCacheEvent::BlockRemoved { block_hashes, medium, + extra_key, }); } } @@ -5927,13 +5931,13 @@ pub enum KvCacheEvent { medium: StorageMedium, cache_salt: Option>, session_id: Option>, - /// Set only in the dynamo format, and not published. Python uses it to - /// find the LoRA adapter name. + /// Not published; Python maps it to the LoRA name that namespaces the hashes. extra_key: Option>, }, BlockRemoved { block_hashes: Vec, medium: StorageMedium, + extra_key: Option>, }, AllBlocksCleared, } diff --git a/test/registered/unit/disaggregation/test_kv_events.py b/test/registered/unit/disaggregation/test_kv_events.py index c209570b0f..a46362a47b 100644 --- a/test/registered/unit/disaggregation/test_kv_events.py +++ b/test/registered/unit/disaggregation/test_kv_events.py @@ -328,6 +328,11 @@ class TestBlockStoredWireFormat(CustomTestCase): self.assertEqual(decoded["cache_salt"], "tenant-a") self.assertEqual(decoded["session_id"], "session-a") + def test_lora_name_is_a_named_field(self): + event = self._event(lora_name="adapter-a") + decoded = msgspec.msgpack.decode(msgspec.msgpack.encode(event)) + self.assertEqual(decoded["lora_name"], "adapter-a") + def test_one_decoder_reads_a_mixed_batch(self): batch = KVEventBatch( ts=1.0, diff --git a/test/registered/unit/dllm/test_gemma4_uniform_lifecycle.py b/test/registered/unit/dllm/test_gemma4_uniform_lifecycle.py index c254d250f4..ff6f499fd5 100644 --- a/test/registered/unit/dllm/test_gemma4_uniform_lifecycle.py +++ b/test/registered/unit/dllm/test_gemma4_uniform_lifecycle.py @@ -380,6 +380,7 @@ class TestGemma4RequestValidation(unittest.TestCase): scheduler.tokenizer = None scheduler.dllm_config = _Scheduler(fdfo=False).dllm_config scheduler._maybe_namespace_elastic_radix_cache = Mock() + scheduler._record_kv_event_lora_name = Mock() scheduler._add_request_to_queue = Mock() scheduler.init_req_max_new_tokens = Mock() scheduler.output_streamer = Mock() diff --git a/test/registered/unit/managers/test_scheduler_sampling_mask_validation.py b/test/registered/unit/managers/test_scheduler_sampling_mask_validation.py index ebc47b313c..d4f52acaac 100644 --- a/test/registered/unit/managers/test_scheduler_sampling_mask_validation.py +++ b/test/registered/unit/managers/test_scheduler_sampling_mask_validation.py @@ -34,6 +34,7 @@ class TestSchedulerSamplingMaskValidation(CustomTestCase): scheduler.tokenizer = None scheduler.dllm_config = None scheduler._maybe_namespace_elastic_radix_cache = MagicMock() + scheduler._record_kv_event_lora_name = MagicMock() scheduler.spec_algorithm = SimpleNamespace( is_dflash_family=lambda: False, is_uno=lambda: False, diff --git a/test/registered/unit/mem_cache/test_radix_cache_unit.py b/test/registered/unit/mem_cache/test_radix_cache_unit.py index a1dbd5cb06..fa92b32627 100644 --- a/test/registered/unit/mem_cache/test_radix_cache_unit.py +++ b/test/registered/unit/mem_cache/test_radix_cache_unit.py @@ -27,6 +27,7 @@ import random import unittest import unittest.mock from array import array +from types import SimpleNamespace import torch @@ -864,6 +865,52 @@ class TestRadixCache(CustomTestCase): first, branch = [e for e in cache.take_events() if isinstance(e, BlockStored)] self.assertEqual(branch.parent_block_hash, first.block_hashes[0]) + def test_base_request_with_empty_lora_path_keeps_base_hashes(self): + """lora_path="" resolves to no adapter; it must not namespace the + base-model blocks published after it.""" + published = [] + for register_empty in (False, True): + cache = RadixCache.create_simulated( + page_size=2, enable_kv_cache_events=True + ) + if register_empty: + base_req = SimpleNamespace(extra_key=None, lora_id=None) + cache.kv_events.lora_names.register(base_req, "") + cache.insert(InsertParams(key=RadixKey(array("q", [1, 2, 3, 4])))) + published.append( + [ + (event.lora_name, tuple(event.block_hashes)) + for event in cache.take_events() + if isinstance(event, BlockStored) + ] + ) + self.assertEqual(published[0], published[1]) + + def test_loaded_adapter_id_does_not_namespace_a_base_request(self): + """A base request's user extra_key cannot borrow a loaded adapter's hashes.""" + extra_key = "0123456789abcdef0123456789abcdef" + published = [] + for namespace in (None, extra_key): + cache = RadixCache.create_simulated( + page_size=2, enable_kv_cache_events=True + ) + cache.kv_events.lora_names.register( + lora_id=extra_key, lora_name="adapter-a" + ) + cache.insert( + InsertParams( + key=RadixKey(array("q", [1, 2, 3, 4]), extra_key=namespace) + ) + ) + published.append( + [ + (event.lora_name, tuple(event.block_hashes)) + for event in cache.take_events() + if isinstance(event, BlockStored) + ] + ) + self.assertEqual(published[0], published[1]) + def test_cache_salt_event_hashes_are_preserved_across_node_split(self): cache = RadixCache.create_simulated(page_size=2, enable_kv_cache_events=True) original = RadixKey(array("q", [1, 2, 3, 4]), cache_salt="tenant-a") diff --git a/test/registered/unit/mem_cache/test_unified_radix_cache_unittest.py b/test/registered/unit/mem_cache/test_unified_radix_cache_unittest.py index 203dbe3ac9..435b141679 100644 --- a/test/registered/unit/mem_cache/test_unified_radix_cache_unittest.py +++ b/test/registered/unit/mem_cache/test_unified_radix_cache_unittest.py @@ -25,6 +25,8 @@ from sglang.srt.disaggregation.kv_events import ( BlockRemoved, BlockStored, StorageMedium, + kv_event_namespace_seed, + namespaced_block_hash, ) from sglang.srt.environ import envs from sglang.srt.managers.schedule_batch import ( @@ -50,6 +52,7 @@ from sglang.srt.mem_cache.base_prefix_cache import ( ) from sglang.srt.mem_cache.cache_init_params import CacheInitParams from sglang.srt.mem_cache.common import available_and_evictable_str, release_kv_cache +from sglang.srt.mem_cache.events import KvEventLoraNames from sglang.srt.mem_cache.hicache_storage import ( PoolHitPolicy, PoolName, @@ -601,6 +604,7 @@ def _aux_storage_key_transfers(cache, node_id): def build_fixture( cfg: CacheConfig, enable_kv_cache_events: bool = False, + dynamo_kv_event_format: bool = False, enable_session_radix_cache: bool = False, tree_page_size: Optional[int] = None, mamba_cache_chunk_size: Optional[int] = None, @@ -608,6 +612,7 @@ def build_fixture( dict[ComponentType, type[TreeComponent]] ] = None, tree_core_backend: Optional[str] = None, + kv_event_lora_names: Optional[KvEventLoraNames] = None, ): """Create (tree, allocator, req_to_token_pool) from a CacheConfig. @@ -738,6 +743,8 @@ def build_fixture( tree_components=cfg.components, enable_mamba_extra_buffer=cfg.enable_mamba_extra_buffer, enable_kv_cache_events=enable_kv_cache_events, + dynamo_kv_event_format=dynamo_kv_event_format, + kv_event_lora_names=kv_event_lora_names or KvEventLoraNames(), enable_session_radix_cache=enable_session_radix_cache, eviction_policy=cfg.eviction_policy, is_eagle=cfg.is_eagle, @@ -1295,6 +1302,23 @@ class TestUnifiedRadixCacheEagleHiCacheStorageKey(CustomTestCase): cache.sanity_check() +class _LoraReq: + """The request fields KvEventLoraNames reads.""" + + def __init__(self, extra_key, lora_id="lora-id"): + self.extra_key = extra_key + self.lora_id = lora_id + + +def _lora_names(**names_by_extra_key): + """A name table plus the requests that keep its entries alive.""" + table = KvEventLoraNames() + reqs = [_LoraReq(key) for key in names_by_extra_key] + for req in reqs: + table.register(req, names_by_extra_key[req.extra_key]) + return table, reqs + + class TestUnifiedRadixCacheKVEvents(CustomTestCase): cfg = CacheConfig(page_size=2, kv_size=64, max_context_len=64) @@ -1521,6 +1545,159 @@ class TestUnifiedRadixCacheKVEvents(CustomTestCase): stored_hashes(extra_key="adapter-b", cache_salt="tenant-a"), ) + def test_lora_external_ids_separate_adapter_loads(self): + """External IDs distinguish loads; routers match tokens and public names.""" + lora_names, _reqs = _lora_names( + **{ + "uuid-a": "adapter-a", + "user|uuid-a2": "adapter-a", + "uuid-b": "adapter-b", + } + ) + + def store_then_evict(extra_key): + cache, allocator, _ = build_fixture( + self.cfg, enable_kv_cache_events=True, kv_event_lora_names=lora_names + ) + cache.take_events() + self._insert(cache, allocator, [1, 2, 3, 4], extra_key=extra_key) + stored = self._stored_events(cache, StorageMedium.GPU) + cache.evict(EvictParams(num_tokens=4)) + removed = self._removed_events(cache, StorageMedium.GPU) + self.assertEqual(self._event_hashes(removed), self._event_hashes(stored)) + return stored + + base = store_then_evict(None) + lora_a = store_then_evict("uuid-a") + self.assertIsNone(base[0].lora_name) + self.assertEqual(lora_a[0].lora_name, "adapter-a") + seed = kv_event_namespace_seed(extra_key="uuid-a", cache_salt=None) + self.assertEqual( + self._event_hashes(lora_a), + [ + namespaced_block_hash(h, namespace_seed=seed) + for h in self._event_hashes(base) + ], + ) + # Another load of the same public name has distinct external IDs. + self.assertNotEqual( + self._event_hashes(store_then_evict("user|uuid-a2")), + self._event_hashes(lora_a), + ) + self.assertNotEqual( + self._event_hashes(store_then_evict("uuid-b")), + self._event_hashes(lora_a), + ) + + def test_dynamo_lora_hashes_keep_the_dynamo_namespace(self): + """The default LoRA mix must not be applied a second time to Dynamo hashes.""" + extra_key = "0123456789abcdef0123456789abcdef" + names, reqs = _lora_names(**{extra_key: "adapter-a"}) + + def publish(*, dynamo_format, namespace): + cache, allocator, _ = build_fixture( + self.cfg, + enable_kv_cache_events=True, + dynamo_kv_event_format=dynamo_format, + kv_event_lora_names=names, + ) + cache.take_events() + self._insert( + cache, + allocator, + [1, 2, 3, 4], + extra_key=namespace, + cache_salt="tenant-a", + ) + stored = self._stored_events(cache, StorageMedium.GPU) + cache.evict(EvictParams(num_tokens=4)) + removed = self._removed_events(cache, StorageMedium.GPU) + self.assertEqual(self._event_hashes(removed), self._event_hashes(stored)) + return stored + + base = publish(dynamo_format=False, namespace=None) + lora = publish(dynamo_format=True, namespace=extra_key) + seed = kv_event_namespace_seed(extra_key=extra_key, cache_salt="tenant-a") + self.assertEqual( + self._event_hashes(lora), + [ + namespaced_block_hash(h, namespace_seed=seed) + for h in self._event_hashes(base) + ], + ) + self.assertEqual(lora[0].lora_name, "adapter-a") + + def test_lora_request_without_optional_ipc_name_uses_loaded_adapter(self): + names = KvEventLoraNames() + names.register(lora_id="loaded-id", lora_name="adapter-a") + req = _LoraReq("user|loaded-id", lora_id="loaded-id") + names.register(req) + cache, allocator, _ = build_fixture( + self.cfg, enable_kv_cache_events=True, kv_event_lora_names=names + ) + cache.take_events() + self._insert(cache, allocator, [1, 2, 3, 4], extra_key=req.extra_key) + stored = self._stored_events(cache, StorageMedium.GPU) + self.assertEqual(stored[0].lora_name, "adapter-a") + cache.evict(EvictParams(num_tokens=4)) + self.assertEqual( + self._event_hashes(self._removed_events(cache, StorageMedium.GPU)), + self._event_hashes(stored), + ) + + def test_lora_name_survives_a_gap_before_a_delayed_store(self): + """Write-back can publish a store after the request and GPU blocks are gone.""" + lora_names, reqs = _lora_names(**{"uuid-a": "adapter-a"}) + cache, allocator, _ = build_fixture( + self.cfg, enable_kv_cache_events=True, kv_event_lora_names=lora_names + ) + cache.take_events() + self._insert(cache, allocator, [1, 2, 3, 4], extra_key="uuid-a") + del reqs + stored = self._stored_events(cache, StorageMedium.GPU) + self.assertEqual(lora_names.get("uuid-a"), "adapter-a") + + cache.evict(EvictParams(num_tokens=4)) + removed = self._removed_events(cache, StorageMedium.GPU) + self.assertEqual(self._event_hashes(removed), self._event_hashes(stored)) + self.assertEqual(lora_names.get("uuid-a"), "adapter-a") + # A late background store needs no new request/name registration. + self._insert(cache, allocator, [1, 2, 3, 4], extra_key="uuid-a") + delayed = self._stored_events(cache, StorageMedium.GPU) + self.assertEqual(delayed[0].lora_name, "adapter-a") + self.assertEqual(self._event_hashes(delayed), self._event_hashes(stored)) + + def test_lora_name_survives_until_queued_stores_are_taken(self): + """A finished request's blocks still publish under its adapter when + other requests register before the events are taken.""" + lora_names, reqs = _lora_names(**{"uuid-a": "adapter-a"}) + cache, allocator, _ = build_fixture( + self.cfg, enable_kv_cache_events=True, kv_event_lora_names=lora_names + ) + cache.take_events() + self._insert(cache, allocator, [1, 2, 3, 4], extra_key="uuid-a") + del reqs + for i in range(200): + lora_names.register(_LoraReq(f"other-{i}"), "adapter-b") + + stored = self._stored_events(cache, StorageMedium.GPU) + self.assertEqual([e.lora_name for e in stored], ["adapter-a"]) + + def test_lora_event_parentage_survives_node_split(self): + lora_names, _reqs = _lora_names(**{"uuid-a": "adapter-a"}) + cache, allocator, _ = build_fixture( + self.cfg, enable_kv_cache_events=True, kv_event_lora_names=lora_names + ) + cache.take_events() + + self._insert(cache, allocator, [1, 2, 3, 4], extra_key="uuid-a") + original = self._stored_events(cache, StorageMedium.GPU) + self._insert(cache, allocator, [1, 2, 5, 6], extra_key="uuid-a") + branch = self._stored_events(cache, StorageMedium.GPU) + self.assertEqual(len(branch), 1) + self.assertEqual(branch[0].lora_name, "adapter-a") + self.assertEqual(branch[0].parent_block_hash, original[0].block_hashes[0]) + def test_kv_events_split_preserves_block_hash_parentage(self): cache, allocator, _ = build_fixture(self.cfg, enable_kv_cache_events=True) cache.take_events() # Clear the reset event. diff --git a/test/registered/unit/multimodal/test_gpu_feature_transport.py b/test/registered/unit/multimodal/test_gpu_feature_transport.py index f67d717a7d..a29e580e13 100644 --- a/test/registered/unit/multimodal/test_gpu_feature_transport.py +++ b/test/registered/unit/multimodal/test_gpu_feature_transport.py @@ -982,6 +982,7 @@ class TestSchedulerMmTransportBoundary(CustomTestCase): scheduler.tokenizer = object() scheduler.disable_radix_cache = False scheduler._maybe_namespace_elastic_radix_cache = MagicMock() + scheduler._record_kv_event_lora_name = MagicMock() scheduler._add_request_to_queue = MagicMock() scheduler._get_multimodal_inputs = MagicMock( side_effect=scheduler_module._MultimodalInputProcessingError("bad image") @@ -998,6 +999,7 @@ class TestSchedulerMmTransportBoundary(CustomTestCase): priority=None, dimensions=None, lora_id=None, + lora_name=None, http_worker_ipc=None, time_stats=None, return_pooled_hidden_states=False,