From d62b9c2480f3b4b2ff16344c90e6a73e4b1d09fa Mon Sep 17 00:00:00 2001 From: Banruo Liu Date: Sun, 11 Oct 2026 10:23:30 -0500 Subject: [PATCH] [KV Events] Publish LoRA adapter names and namespaced block hashes (#42455) Signed-off-by: Banruo Liu Co-authored-by: Shangming Cai --- python/sglang/srt/disaggregation/kv_events.py | 56 +++++++- python/sglang/srt/managers/scheduler.py | 19 +++ .../sglang/srt/mem_cache/cache_init_params.py | 4 + python/sglang/srt/mem_cache/events.py | 81 ++++++++++- .../sglang/srt/mem_cache/kv_cache_builder.py | 5 + python/sglang/srt/mem_cache/radix_cache.py | 8 +- .../srt/mem_cache/rust_tree_core/adapter.py | 19 ++- .../unified_cache/unified_tree_core.py | 5 +- rust/sglang-radix-tree/src/node.rs | 6 + rust/sglang-radix-tree/src/python_bindings.rs | 8 +- .../src/tests/unified_tree_core.rs | 131 ++++++++++++++++++ .../src/unified_tree_core.rs | 98 +++++++++++-- .../unit/disaggregation/test_kv_events.py | 16 +++ .../unit/mem_cache/test_radix_cache_unit.py | 64 ++++++++- .../test_rust_tree_core_integration.py | 52 +++++++ 15 files changed, 546 insertions(+), 26 deletions(-) diff --git a/python/sglang/srt/disaggregation/kv_events.py b/python/sglang/srt/disaggregation/kv_events.py index 18dfe7cf7d..0597c72dfd 100644 --- a/python/sglang/srt/disaggregation/kv_events.py +++ b/python/sglang/srt/disaggregation/kv_events.py @@ -19,6 +19,7 @@ KV caching events import atexit import enum +import hashlib import logging import os import queue @@ -28,7 +29,7 @@ from abc import ABC, abstractmethod from collections import deque from itertools import count from queue import Queue -from typing import Any, Callable, Optional, Union +from typing import Any, Callable, Literal, Optional, Union import msgspec import zmq @@ -289,6 +290,9 @@ class BlockStored(KVCacheEvent): # Session that triggered this store. Attribution only: the blocks may be # shared with other sessions, and the hash does not depend on it. session_id: Optional[str] = None + # The LoRA adapter name of the request that stored these blocks. + # Set only in the dynamo format. + lora_name: Optional[str] = None class BlockRemoved(KVCacheEvent): @@ -300,6 +304,42 @@ class AllBlocksCleared(KVCacheEvent): pass +# The dynamo format publishes namespaced hashes as block hashes. If you change +# this tag, all of these hashes change. unified_tree_core.rs has a Rust copy. +_NAMESPACE_SEED_TAG = b"sglang-kv-event-namespace-v1" + + +def kv_event_namespace_seed( + *, extra_key: Optional[str], cache_salt: Optional[str] +) -> Optional[bytes]: + """Return the seed for namespaced_block_hash, or None for no namespace.""" + if extra_key is None and cache_salt is None: + return None + digest = hashlib.sha256(_NAMESPACE_SEED_TAG) + # The presence byte and the length prefix make ("a", "bc") and ("ab", "c") differ. + for part in (extra_key, cache_salt): + if part is None: + digest.update(b"\x00") + continue + encoded = part.encode("utf-8") + digest.update(b"\x01" + len(encoded).to_bytes(8, "little") + encoded) + return digest.digest() + + +def namespaced_block_hash(block_hash: int, *, namespace_seed: Optional[bytes]) -> int: + """Return the namespaced hash of a published block hash. + + If namespace_seed is None, the result is ``block_hash``. If not, the result is + the first 8 bytes of SHA-256(seed, block_hash as 8 big-endian bytes), as int64. + """ + if namespace_seed is None: + return block_hash + digest = hashlib.sha256( + namespace_seed + block_hash.to_bytes(8, "big", signed=True) + ).digest() + return int.from_bytes(digest[:8], "big", signed=True) + + class KVEventBatch(EventBatch): events: list[Union[BlockStored, BlockRemoved, AllBlocksCleared]] @@ -680,12 +720,24 @@ class KVEventsConfig(BaseModel): this topic to receive events. """ + format: Literal["default", "dynamo"] = "default" + """The event format. "default" is the SGLang format. "dynamo" is the format + the Dynamo KV router reads: block hashes also depend on extra_key and + cache_salt (see ``namespaced_block_hash``), and BlockStored has lora_name. + """ + @classmethod def from_cli(cls, cli_value: str) -> "KVEventsConfig": """Parse the CLI value for the event publisher config.""" return KVEventsConfig.model_validate_json(cli_value) +def uses_dynamo_format(kv_events_config: Optional[str]) -> bool: + if not kv_events_config: + return False + return KVEventsConfig.from_cli(kv_events_config).format == "dynamo" + + class EventPublisherFactory: _registry: dict[str, Callable[..., EventPublisher]] = { "null": NullEventPublisher, @@ -705,6 +757,8 @@ class EventPublisherFactory: return NullEventPublisher() config = KVEventsConfig.from_cli(config) config_dict = config.model_dump() + # The cache event recorder uses the format. The publisher does not accept it. + config_dict.pop("format") kind = config_dict.pop("publisher", "null") try: diff --git a/python/sglang/srt/managers/scheduler.py b/python/sglang/srt/managers/scheduler.py index ca6a0ad21f..13c88720b4 100644 --- a/python/sglang/srt/managers/scheduler.py +++ b/python/sglang/srt/managers/scheduler.py @@ -87,6 +87,7 @@ from sglang.srt.disaggregation.decode_kvcache_offload_manager import ( DecodeKVCacheOffloadManager, ) from sglang.srt.disaggregation.encoder.receiver import create_mm_receiver +from sglang.srt.disaggregation.kv_events import uses_dynamo_format from sglang.srt.disaggregation.prefill import ( PrefillBootstrapQueue, SchedulerDisaggregationPrefillMixin, @@ -297,6 +298,7 @@ from sglang.srt.mem_cache.common import ( discard_kv_cache_backup, release_kv_cache, ) +from sglang.srt.mem_cache.events import LoRANameTable from sglang.srt.model_executor.forward_batch_info import PPProxyTensors from sglang.srt.model_executor.runner_utils.pool import prewarm_graph_pool_borrow from sglang.srt.model_loader.utils import get_resolved_model_impl @@ -564,6 +566,7 @@ class Scheduler( time.sleep(t) # Init cache and memory pool + self.init_kv_event_lora_names() result = kv_cache_builder.build_kv_cache( server_args=self.server_args, model_config=self.model_config, @@ -577,6 +580,10 @@ class Scheduler( and get_parallel().attn_tp_rank == 0 and get_parallel().attn_cp_rank == 0 ), + dynamo_kv_event_format=uses_dynamo_format( + get_observability().kv_events_config + ), + kv_event_lora_names=self.kv_event_lora_names, enable_hierarchical_cache=self.enable_hierarchical_cache, hicache_draft_plan=( self.draft_worker.hicache_draft_plan @@ -2429,6 +2436,10 @@ class Scheduler( groups += [self.pp_group] rank_consensus_checker.configure(groups) + def init_kv_event_lora_names(self) -> None: + """Init the table that maps lora_id to adapter name for KV events.""" + self.kv_event_lora_names = LoRANameTable.from_lora_refs(get_lora().lora_paths) + def init_kv_events_publisher(self) -> None: self.kv_events_publisher = SchedulerKvEventsPublisher( kv_events_config=get_observability().kv_events_config, @@ -5768,6 +5779,10 @@ class Scheduler( """In-place loading a new lora adapter from disk or huggingface.""" result = self.tp_worker.load_lora_adapter(recv_req) + if result.success: + self.kv_event_lora_names.register( + lora_id=recv_req.lora_id, lora_name=recv_req.lora_name + ) return result def load_lora_adapter_from_tensors( @@ -5776,6 +5791,10 @@ class Scheduler( """In-place loading a new lora adapter from serialized tensors.""" result = self.tp_worker.load_lora_adapter_from_tensors(recv_req) + if result.success: + self.kv_event_lora_names.register( + lora_id=recv_req.lora_id, lora_name=recv_req.lora_name + ) return result def unload_lora_adapter( diff --git a/python/sglang/srt/mem_cache/cache_init_params.py b/python/sglang/srt/mem_cache/cache_init_params.py index 0aa5d1e4d6..7722516d2a 100644 --- a/python/sglang/srt/mem_cache/cache_init_params.py +++ b/python/sglang/srt/mem_cache/cache_init_params.py @@ -7,6 +7,7 @@ import torch if TYPE_CHECKING: from sglang.srt.mem_cache.allocator import BaseTokenToKVPoolAllocator + from sglang.srt.mem_cache.events import LoRANameTable from sglang.srt.mem_cache.memory_pool import ReqToTokenPool from sglang.srt.mem_cache.unified_cache.components import ComponentType from sglang.srt.mem_cache.unified_cache.components.base import ( @@ -33,6 +34,9 @@ class CacheInitParams: enable_metrics: bool = False enable_kv_cache_events: bool = False + dynamo_kv_event_format: bool = False + # The scheduler holds the same table and adds each adapter when it loads. + kv_event_lora_names: Optional[LoRANameTable] = None enable_session_radix_cache: bool = False enable_mamba_extra_buffer: bool = False diff --git a/python/sglang/srt/mem_cache/events.py b/python/sglang/srt/mem_cache/events.py index c5f7c341cb..cf7d8c5d4b 100644 --- a/python/sglang/srt/mem_cache/events.py +++ b/python/sglang/srt/mem_cache/events.py @@ -18,13 +18,17 @@ consumed by KV-aware routers (e.g. dynamo). A cache holds one recorder and calls it; the recorder owns the queue and needs nothing back from its owner. """ -from typing import Any, Optional +from __future__ import annotations + +from typing import TYPE_CHECKING, Any, Iterable, Optional from sglang.srt.disaggregation.kv_events import ( AllBlocksCleared, BlockRemoved, BlockStored, StorageMedium, + kv_event_namespace_seed, + namespaced_block_hash, ) from sglang.srt.mem_cache.utils import ( compute_node_event_hash_values, @@ -32,6 +36,40 @@ from sglang.srt.mem_cache.utils import ( hash_str_to_int64, ) +if TYPE_CHECKING: + from sglang.srt.lora.lora_registry import LoRARef + +# Length of LoRARef.lora_id, a uuid4 or uuid5 hex string. Req adds the id to the +# end of extra_key, so the last 32 characters of extra_key are the id. +_LORA_ID_LEN = 32 + + +class LoRANameTable: + """Find the LoRA adapter name from the lora_id at the end of extra_key. + + Req adds lora_id to the end of extra_key, so a tree node has only the id. + The table keeps all entries. A lora_id is not used again, and nodes of an + unloaded adapter can publish stores until eviction. + """ + + def __init__(self): + self._names: dict[str, str] = {} + + @classmethod + def from_lora_refs(cls, lora_refs: Optional[Iterable[LoRARef]]) -> LoRANameTable: + table = cls() + for lora_ref in lora_refs or (): + table.register(lora_id=lora_ref.lora_id, lora_name=lora_ref.lora_name) + return table + + def register(self, *, lora_id: str, lora_name: str) -> None: + self._names[lora_id] = lora_name + + def resolve(self, extra_key: Optional[str]) -> Optional[str]: + if not extra_key: + return None + return self._names.get(extra_key[-_LORA_ID_LEN:]) + class KVCacheEventRecorder: """Collects KV placement events for one cache. @@ -40,9 +78,18 @@ class KVCacheEventRecorder: empty list, so callers never have to guard. """ - def __init__(self, *, enabled: bool, page_size: int): + def __init__( + self, + *, + enabled: bool, + page_size: int, + dynamo_format: bool = False, + lora_names: Optional[LoRANameTable] = None, + ): self.enabled = enabled self.page_size = page_size + self.dynamo_format = dynamo_format + self.lora_names = lora_names if lora_names is not None else LoRANameTable() self._queue: list = [] def enqueue(self, event) -> None: @@ -103,6 +150,14 @@ class KVCacheEventRecorder: return None return hash_str_to_int64(parent_hash_values[-1]) + def _namespace_seed(self, node: Any) -> Optional[bytes]: + # The default format publishes the hashes unchanged. + if not self.dynamo_format: + return None + return kv_event_namespace_seed( + extra_key=node.key.extra_key, cache_salt=node.key.cache_salt + ) + def record_store( self, node: Any, medium=None, *, session_id: Optional[str] = None ) -> None: @@ -115,7 +170,15 @@ class KVCacheEventRecorder: medium = StorageMedium.GPU event_hash_values = self._node_event_hash_values(node) + namespace_seed = self._namespace_seed(node) parent_block_hash = self._parent_block_hash(node) + if parent_block_hash is not None: + parent_block_hash = namespaced_block_hash( + parent_block_hash, namespace_seed=namespace_seed + ) + lora_name = ( + self.lora_names.resolve(node.key.extra_key) if self.dynamo_format else None + ) page_index = 0 logical_len = len(node.key) @@ -131,7 +194,10 @@ class KVCacheEventRecorder: else: page_tokens = list(raw[start:end]) - block_hash = hash_str_to_int64(event_hash_values[page_index]) + block_hash = namespaced_block_hash( + hash_str_to_int64(event_hash_values[page_index]), + namespace_seed=namespace_seed, + ) self.enqueue( BlockStored( @@ -143,6 +209,7 @@ class KVCacheEventRecorder: medium=medium, cache_salt=node.key.cache_salt, session_id=session_id, + lora_name=lora_name, ) ) @@ -160,6 +227,7 @@ class KVCacheEventRecorder: # Hash values must match what was stored. event_hash_values = self._node_event_hash_values(node) + namespace_seed = self._namespace_seed(node) block_hashes = [] logical_len = len(node.key) @@ -169,7 +237,12 @@ class KVCacheEventRecorder: if end <= start: continue - block_hashes.append(hash_str_to_int64(event_hash_values[page_index])) + block_hashes.append( + namespaced_block_hash( + hash_str_to_int64(event_hash_values[page_index]), + namespace_seed=namespace_seed, + ) + ) page_index += 1 if block_hashes: diff --git a/python/sglang/srt/mem_cache/kv_cache_builder.py b/python/sglang/srt/mem_cache/kv_cache_builder.py index 43830ce3b3..4158412db9 100644 --- a/python/sglang/srt/mem_cache/kv_cache_builder.py +++ b/python/sglang/srt/mem_cache/kv_cache_builder.py @@ -61,6 +61,7 @@ from sglang.srt.utils import ceil_align, is_hip if TYPE_CHECKING: from sglang.srt.configs.model_config import ModelConfig from sglang.srt.managers.tp_worker import BaseTpWorker + from sglang.srt.mem_cache.events import LoRANameTable from sglang.srt.server_args import ServerArgs from sglang.srt.speculative.base_spec_worker import HiCacheDraftPlan from sglang.srt.speculative.spec_info import SpeculativeAlgorithm @@ -255,6 +256,8 @@ def build_kv_cache( enable_kv_cache_events: bool, enable_hierarchical_cache: bool, hicache_draft_plan: Optional[HiCacheDraftPlan] = None, + dynamo_kv_event_format: bool = False, + kv_event_lora_names: Optional[LoRANameTable] = None, ) -> KVCacheBuildResult: parallel = get_parallel() sliding_window_size: Optional[int] = None @@ -346,6 +349,8 @@ def build_kv_cache( eviction_policy_config=get_memory().radix_eviction_policy_config, enable_metrics=enable_metrics, enable_kv_cache_events=enable_kv_cache_events, + dynamo_kv_event_format=dynamo_kv_event_format, + kv_event_lora_names=kv_event_lora_names, enable_session_radix_cache=get_memory().enable_session_radix_cache, enable_mamba_extra_buffer=get_exec().mamba.enable_mamba_extra_buffer, enable_mamba_extra_buffer_lazy=get_exec().mamba.enable_mamba_extra_buffer_lazy, diff --git a/python/sglang/srt/mem_cache/radix_cache.py b/python/sglang/srt/mem_cache/radix_cache.py index 71ba4a4357..523d51757e 100644 --- a/python/sglang/srt/mem_cache/radix_cache.py +++ b/python/sglang/srt/mem_cache/radix_cache.py @@ -45,7 +45,7 @@ from sglang.srt.mem_cache.base_prefix_cache import ( MatchPrefixParams, MatchResult, ) -from sglang.srt.mem_cache.events import KVCacheEventRecorder +from sglang.srt.mem_cache.events import KVCacheEventRecorder, LoRANameTable from sglang.srt.mem_cache.utils import ( get_eviction_strategy, split_node_hash_value, @@ -292,6 +292,8 @@ class RadixCache(BasePrefixCache): self.kv_events = KVCacheEventRecorder( enabled=params.enable_kv_cache_events and not self.disable, page_size=self.page_size, + dynamo_format=params.dynamo_kv_event_format, + lora_names=params.kv_event_lora_names, ) if params.enable_metrics: @@ -321,6 +323,8 @@ class RadixCache(BasePrefixCache): mock_allocator: Optional[Any] = None, page_size: int = 1, enable_kv_cache_events: bool = False, + dynamo_kv_event_format: bool = False, + kv_event_lora_names: Optional[LoRANameTable] = None, ) -> RadixCache: """Init a radix cache without memory pools for simulation purpose.""" params = CacheInitParams( @@ -329,6 +333,8 @@ class RadixCache(BasePrefixCache): token_to_kv_pool_allocator=mock_allocator, page_size=page_size, enable_kv_cache_events=enable_kv_cache_events, + dynamo_kv_event_format=dynamo_kv_event_format, + kv_event_lora_names=kv_event_lora_names, ) return RadixCache(params) diff --git a/python/sglang/srt/mem_cache/rust_tree_core/adapter.py b/python/sglang/srt/mem_cache/rust_tree_core/adapter.py index 5132035cfd..7b6ba48cd1 100644 --- a/python/sglang/srt/mem_cache/rust_tree_core/adapter.py +++ b/python/sglang/srt/mem_cache/rust_tree_core/adapter.py @@ -26,6 +26,7 @@ from sglang.srt.mem_cache.base_prefix_cache import ( MatchPrefixParams, MatchResult, ) +from sglang.srt.mem_cache.events import LoRANameTable from sglang.srt.mem_cache.hicache_storage import PoolHitPolicy, PoolName, PoolTransfer from sglang.srt.mem_cache.radix_cache import RadixKey from sglang.srt.mem_cache.rust_tree_core.extension import bindings @@ -95,7 +96,7 @@ def _radix_key_buffer(key: RadixKey) -> array: return token_ids -def _kv_event_from_tagged(event: tuple): +def _kv_event_from_tagged(event: tuple, lora_names: LoRANameTable): """Build the Python KV cache event for one of the binding's tagged tuples.""" tag = event[0] if tag == "block_stored": @@ -108,6 +109,7 @@ def _kv_event_from_tagged(event: tuple): medium=StorageMedium(event[5]), cache_salt=event[6], session_id=event[7], + lora_name=lora_names.resolve(event[8]), ) if tag == "block_removed": return BlockRemoved(block_hashes=event[1], medium=StorageMedium(event[2])) @@ -301,15 +303,21 @@ def _fill_evict_result(binding_result, result): class _RustKVCacheEventRecorder: """Expose the Rust event queue through the Python recorder interface.""" - def __init__(self, binding, enabled: bool): + def __init__( + self, binding, enabled: bool, lora_names: Optional[LoRANameTable] = None + ): self._binding = binding self.enabled = enabled + self.lora_names = lora_names if lora_names is not None else LoRANameTable() def record_all_cleared(self) -> None: self._binding.record_all_cleared_event() def take(self) -> list: - return [_kv_event_from_tagged(event) for event in self._binding.take_events()] + return [ + _kv_event_from_tagged(event, self.lora_names) + for event in self._binding.take_events() + ] class RustUnifiedTreeCore(UnifiedTreeCoreInterface): @@ -422,6 +430,7 @@ class RustUnifiedTreeCore(UnifiedTreeCoreInterface): swa_sliding_window_size=params.sliding_window_size, swa_req_ring=is_swa_req_ring(self._allocator), enable_kv_cache_events=params.enable_kv_cache_events, + dynamo_kv_event_format=params.dynamo_kv_event_format, mamba_cache_chunk_size=( mamba_cache_chunk_size() if has_mamba else None ), @@ -434,7 +443,9 @@ class RustUnifiedTreeCore(UnifiedTreeCoreInterface): [int(component) for component in self.tree_components], ) self.kv_events = _RustKVCacheEventRecorder( - self._binding, params.enable_kv_cache_events + self._binding, + params.enable_kv_cache_events, + lora_names=params.kv_event_lora_names, ) # The default-root empty result, prebuilt once from the binding. self._empty_match_result = _match_result_from_binding( diff --git a/python/sglang/srt/mem_cache/unified_cache/unified_tree_core.py b/python/sglang/srt/mem_cache/unified_cache/unified_tree_core.py index bca490ea9f..4f989087b4 100644 --- a/python/sglang/srt/mem_cache/unified_cache/unified_tree_core.py +++ b/python/sglang/srt/mem_cache/unified_cache/unified_tree_core.py @@ -460,7 +460,10 @@ class UnifiedTreeCore(UnifiedTreeCoreInterface): ) self.kv_events = KVCacheEventRecorder( - enabled=params.enable_kv_cache_events, page_size=self.page_size + enabled=params.enable_kv_cache_events, + page_size=self.page_size, + dynamo_format=params.dynamo_kv_event_format, + lora_names=params.kv_event_lora_names, ) self.reset() diff --git a/rust/sglang-radix-tree/src/node.rs b/rust/sglang-radix-tree/src/node.rs index 492d83f4c1..2c21f6ec35 100644 --- a/rust/sglang-radix-tree/src/node.rs +++ b/rust/sglang-radix-tree/src/node.rs @@ -112,6 +112,12 @@ impl KeyNamespace { .as_ref() .and_then(|namespace| namespace.cache_salt.clone()) } + + pub fn extra_key_arc(&self) -> Option> { + self.0 + .as_ref() + .and_then(|namespace| namespace.extra_key.clone()) + } } impl PartialEq for KeyNamespace { diff --git a/rust/sglang-radix-tree/src/python_bindings.rs b/rust/sglang-radix-tree/src/python_bindings.rs index c90c183577..04456370a8 100644 --- a/rust/sglang-radix-tree/src/python_bindings.rs +++ b/rust/sglang-radix-tree/src/python_bindings.rs @@ -497,6 +497,7 @@ pub struct TreeCoreInitParamsBinding { pub swa_sliding_window_size: Option, pub swa_req_ring: bool, pub enable_kv_cache_events: bool, + pub dynamo_kv_event_format: bool, pub mamba_cache_chunk_size: Option, pub mamba_max_states_per_path: Option, } @@ -519,6 +520,7 @@ impl TreeCoreInitParamsBinding { // Wired post-construction via set_has_swa_host_pool. has_swa_host_pool: false, enable_kv_cache_events: self.enable_kv_cache_events, + dynamo_kv_event_format: self.dynamo_kv_event_format, mamba_cache_chunk_size: self.mamba_cache_chunk_size, mamba_max_states_per_path: self.mamba_max_states_per_path, }) @@ -528,7 +530,7 @@ impl TreeCoreInitParamsBinding { #[pymethods] impl TreeCoreInitParamsBinding { #[new] - #[pyo3(signature = (eviction_policy = "lru".to_string(), page_size = 1, is_write_back = false, enable_hicache = false, write_through_threshold = 256, device = "cpu".to_string(), swa_sliding_window_size = None, enable_kv_cache_events = false, mamba_cache_chunk_size = None, mamba_max_states_per_path = None, slru_protected_threshold = 2, swa_req_ring = false, tlru_tail_budget = 0, tlru_float_config = None))] + #[pyo3(signature = (eviction_policy = "lru".to_string(), page_size = 1, is_write_back = false, enable_hicache = false, write_through_threshold = 256, device = "cpu".to_string(), swa_sliding_window_size = None, enable_kv_cache_events = false, mamba_cache_chunk_size = None, mamba_max_states_per_path = None, slru_protected_threshold = 2, swa_req_ring = false, tlru_tail_budget = 0, tlru_float_config = None, dynamo_kv_event_format = false))] fn new( eviction_policy: String, page_size: usize, @@ -544,6 +546,7 @@ impl TreeCoreInitParamsBinding { swa_req_ring: bool, tlru_tail_budget: usize, tlru_float_config: Option, + dynamo_kv_event_format: bool, ) -> Self { TreeCoreInitParamsBinding { eviction_policy, @@ -558,6 +561,7 @@ impl TreeCoreInitParamsBinding { swa_sliding_window_size, swa_req_ring, enable_kv_cache_events, + dynamo_kv_event_format, mamba_cache_chunk_size, mamba_max_states_per_path, } @@ -2130,6 +2134,7 @@ impl TreeCoreBinding { medium, cache_salt, session_id, + extra_key, } => { let item: Py = ( "block_stored", @@ -2140,6 +2145,7 @@ impl TreeCoreBinding { medium.as_str(), cache_salt.map(|salt| salt.to_string()), session_id.map(|session_id| session_id.to_string()), + extra_key.map(|extra_key| extra_key.to_string()), ) .into_py_any(py)?; list.append(item)?; diff --git a/rust/sglang-radix-tree/src/tests/unified_tree_core.rs b/rust/sglang-radix-tree/src/tests/unified_tree_core.rs index 036694e2d9..5c7d8da76e 100644 --- a/rust/sglang-radix-tree/src/tests/unified_tree_core.rs +++ b/rust/sglang-radix-tree/src/tests/unified_tree_core.rs @@ -3046,6 +3046,7 @@ fn insert_coalesces_parent_linked_block_stores() { medium: StorageMedium::Gpu, cache_salt: None, session_id: None, + extra_key: None, }] ); // Events hash lazily even though the storage tier is off. @@ -3083,6 +3084,7 @@ fn insert_attributes_stored_blocks_to_session_without_changing_hashes() { medium: StorageMedium::Gpu, cache_salt: None, session_id: Some(Arc::from("session-a")), + extra_key: None, }] ); } @@ -3158,6 +3160,7 @@ fn extra_key_nodes_publish_token_only_event_hashes() { medium: StorageMedium::Gpu, cache_salt: None, session_id: None, + extra_key: None, }] ); @@ -3251,6 +3254,7 @@ fn event_coalescing_respects_store_remove_and_clear_boundaries() { medium: StorageMedium::Gpu, cache_salt: None, session_id: None, + extra_key: None, }); assert_eq!(tc.kv_event_queue.len(), 1); // A different block size must not join the parent-linked store tail. @@ -3262,6 +3266,7 @@ fn event_coalescing_respects_store_remove_and_clear_boundaries() { medium: StorageMedium::Gpu, cache_salt: None, session_id: None, + extra_key: None, }); assert_eq!(tc.kv_event_queue.len(), 2); // Matching size and parent are still separated across media. @@ -3273,6 +3278,7 @@ fn event_coalescing_respects_store_remove_and_clear_boundaries() { medium: StorageMedium::Cpu, cache_salt: None, session_id: None, + extra_key: None, }); assert_eq!(tc.kv_event_queue.len(), 3); // Matching size and medium are still separated without the parent link. @@ -3284,6 +3290,7 @@ fn event_coalescing_respects_store_remove_and_clear_boundaries() { medium: StorageMedium::Cpu, cache_salt: None, session_id: None, + extra_key: None, }); assert_eq!(tc.kv_event_queue.len(), 4); tc.enqueue_kv_event_(KvCacheEvent::BlockRemoved { @@ -3327,6 +3334,7 @@ fn event_coalescing_respects_store_remove_and_clear_boundaries() { medium: StorageMedium::Gpu, cache_salt: Some(Arc::from("tenant-a")), session_id: None, + extra_key: None, }); tc.enqueue_kv_event_(KvCacheEvent::BlockStored { block_hashes: vec![2], @@ -3336,6 +3344,7 @@ fn event_coalescing_respects_store_remove_and_clear_boundaries() { medium: StorageMedium::Gpu, cache_salt: Some(Arc::from("tenant-b")), session_id: None, + extra_key: None, }); assert_eq!(tc.kv_event_queue.len(), 2); } @@ -3371,6 +3380,121 @@ fn eviction_emits_block_removed_with_all_page_hashes() { ); } +fn dynamo_events_core(page_size: usize) -> UnifiedTreeCore> { + UnifiedTreeCore::new( + CacheInitParams { + page_size, + enable_kv_cache_events: true, + dynamo_kv_event_format: true, + ..CacheInitParams::default() + }, + vec![FULL], + ) +} + +fn evict_all_device(tc: &mut UnifiedTreeCore>) { + let mut tracker = HashMap::from([(FULL, 0)]); + let (mut device_frees, mut host_frees) = (HashMap::new(), HashMap::new()); + tc.evict_device_start(FULL, 100); + loop { + let (node, step) = tc.evict_device_next_node(FULL, &tracker); + accumulate_step(step, &mut tracker, &mut device_frees, &mut host_frees); + let Some(node) = node else { break }; + let (_, step) = tc + .evict_device_leaf(node, /* is_write_back = */ false) + .expect("live test node"); + accumulate_step(step, &mut tracker, &mut device_frees, &mut host_frees); + } + tc.evict_device_end(FULL); +} + +fn stored_block_hashes(events: &[KvCacheEvent]) -> Vec> { + events + .iter() + .filter_map(|event| match event { + KvCacheEvent::BlockStored { block_hashes, .. } => Some(block_hashes.clone()), + _ => None, + }) + .collect() +} + +#[test] +fn dynamo_format_separates_namespaces_that_share_block_hashes() { + // A router that keys blocks by hash uses these hashes to keep adapter + // blocks and base blocks apart. This is also necessary for removals. + let mut tc = dynamo_events_core(2); + let key = vec![1, 2, 7, 8]; + tc.insert(&insert_params(&key, &[10, 11, 12, 13])); + tc.insert(&insert_params_in_namespace( + &key, + &[20, 21, 22, 23], + Some("lora-a"), + None, + )); + let stored = stored_block_hashes(&tc.take_events()); + let [base, lora] = stored.as_slice() else { + panic!("expected one store per namespace, got {stored:?}"); + }; + assert_ne!(lora, base); + + evict_all_device(&mut tc); + let mut removed: Vec = tc + .take_events() + .into_iter() + .flat_map(|event| match event { + KvCacheEvent::BlockRemoved { block_hashes, .. } => block_hashes, + other => panic!("unexpected event {other:?}"), + }) + .collect(); + let mut expected: Vec = base.iter().chain(lora).copied().collect(); + removed.sort_unstable(); + expected.sort_unstable(); + assert_eq!(removed, expected); +} + +#[test] +fn namespaced_block_hash_is_pinned() { + // Python has its own copy of this function, and its test uses the same values. + // Thus the two copies must give the same hashes, and the hashes must not change. + let seed = kv_event_namespace_seed(&KeyNamespace::new(Some("lora-a"), None)); + assert_eq!( + namespaced_block_hash(seed.as_ref(), 123), + 9220659119954863560 + ); + assert_eq!( + namespaced_block_hash(seed.as_ref(), -5), + -1810974732006142875 + ); +} + +#[test] +fn dynamo_format_parent_link_survives_node_split() { + let mut tc = dynamo_events_core(2); + tc.insert(&insert_params_in_namespace( + &vec![1, 2, 7, 8], + &[10, 11, 12, 13], + Some("lora-a"), + None, + )); + let first = stored_block_hashes(&tc.take_events()); + tc.insert(&insert_params_in_namespace( + &vec![1, 2, 9, 10], + &[10, 11, 14, 15], + Some("lora-a"), + None, + )); + let events = tc.take_events(); + let [ + KvCacheEvent::BlockStored { + parent_block_hash, .. + }, + ] = events.as_slice() + else { + panic!("expected one store for the new branch, got {events:?}"); + }; + assert_eq!(*parent_block_hash, Some(first[0][0])); +} + #[test] fn bigram_insert_events_carry_pair_token_payloads() { let mut tc = UnifiedTreeCore::>::new( @@ -3409,6 +3533,7 @@ fn bigram_insert_events_carry_pair_token_payloads() { medium: StorageMedium::Gpu, cache_salt: None, session_id: None, + extra_key: None, }] ); } @@ -3433,6 +3558,7 @@ fn finish_write_through_emits_cpu_stored_events() { medium: StorageMedium::Cpu, cache_salt: None, session_id: None, + extra_key: None, }] ); } @@ -3489,6 +3615,7 @@ fn load_back_commit_emits_gpu_stored_events() { medium: StorageMedium::Gpu, cache_salt: None, session_id: None, + extra_key: None, }] ); } @@ -3509,6 +3636,7 @@ fn unevict_on_insert_emits_a_gpu_stored_event() { medium: StorageMedium::Gpu, cache_salt: None, session_id: None, + extra_key: None, }] ); } @@ -3598,6 +3726,7 @@ fn split_insert_stores_only_the_new_block_chained_to_the_split_parent() { medium: StorageMedium::Gpu, cache_salt: None, session_id: None, + extra_key: None, }] ); // The split divided the page hashes between the two fragments. @@ -3680,6 +3809,7 @@ fn finish_write_through_after_a_split_publishes_both_fragments() { medium: StorageMedium::Cpu, cache_salt: None, session_id: None, + extra_key: None, }] ); // The matching ack cleared the pending mark on both fragments. @@ -4011,6 +4141,7 @@ fn insert_host_publishes_a_host_store_event() { medium: StorageMedium::Cpu, cache_salt: None, session_id: None, + extra_key: None, }] ); } diff --git a/rust/sglang-radix-tree/src/unified_tree_core.rs b/rust/sglang-radix-tree/src/unified_tree_core.rs index 17bed94236..914ae1a202 100644 --- a/rust/sglang-radix-tree/src/unified_tree_core.rs +++ b/rust/sglang-radix-tree/src/unified_tree_core.rs @@ -546,6 +546,8 @@ pub struct CacheInitParams { pub has_swa_host_pool: bool, /// Whether tree mutations emit BlockStored/BlockRemoved events. pub enable_kv_cache_events: bool, + /// If true, events use the dynamo format: namespaced block hashes and extra_key. + pub dynamo_kv_event_format: bool, /// Chunk alignment for the mamba branching seqlen; None when Mamba is disabled. pub mamba_cache_chunk_size: Option, /// Per-root-path cap on cached Mamba states; None means unlimited. @@ -568,6 +570,7 @@ impl Default for CacheInitParams { swa_req_ring: false, has_swa_host_pool: false, enable_kv_cache_events: false, + dynamo_kv_event_format: false, mamba_cache_chunk_size: None, mamba_max_states_per_path: None, } @@ -637,6 +640,8 @@ pub struct UnifiedTreeCore { pub(crate) has_swa_host_pool: bool, /// Whether tree mutations emit BlockStored/BlockRemoved events. pub(crate) enable_kv_cache_events: bool, + /// If true, events use the dynamo format: namespaced block hashes and extra_key. + pub(crate) dynamo_kv_event_format: bool, /// Queued placement events, drained by take_events. pub(crate) kv_event_queue: Vec>, /// Namespaced event hashes, seeded only by cache_salt; events omit extra_key. @@ -830,6 +835,7 @@ impl UnifiedTreeCore { enable_external_cache_linker: false, has_swa_host_pool: params.has_swa_host_pool, enable_kv_cache_events: params.enable_kv_cache_events, + dynamo_kv_event_format: params.dynamo_kv_event_format, kv_event_queue: Vec::new(), namespaced_event_hashes: HashMap::new(), write_through_threshold: params.write_through_threshold, @@ -3197,6 +3203,7 @@ impl UnifiedTreeCore { medium, cache_salt, session_id, + .. }, ) if *tail_medium == medium && *tail_block_size == block_size @@ -3282,29 +3289,39 @@ impl UnifiedTreeCore { self.arena.node_mut(node_id).hash_value = Some(hash_values); } let cache_salt = self.arena.node(node_id).namespace.cache_salt_arc(); + // Python reads the LoRA adapter name from extra_key in the dynamo format. + let extra_key = self + .dynamo_kv_event_format + .then(|| self.arena.node(node_id).namespace.extra_key_arc()) + .flatten(); let session_id: Option> = session_id.map(Arc::from); let namespaced = self.arena.node(node_id).namespace != KeyNamespace::default(); if namespaced { self.ensure_namespaced_event_hashes_(node_id); } + let namespace_seed = self.namespace_seed_(node_id); let events = { let node = self.arena.node(node_id); - let mut parent_block_hash = node.parent.and_then(|parent_id| { - let parent = self.arena.node(parent_id); - if namespaced { - self.namespaced_event_hashes - .get(&parent.id) - .and_then(|hashes| hashes.last()) - .map(crate::node::hash_digest_to_int64) - } else { - parent - .get_last_hash_value() - .map(crate::node::hash_str_to_int64) - } - }); + let mut parent_block_hash = node + .parent + .and_then(|parent_id| { + let parent = self.arena.node(parent_id); + if namespaced { + self.namespaced_event_hashes + .get(&parent.id) + .and_then(|hashes| hashes.last()) + .map(crate::node::hash_digest_to_int64) + } else { + parent + .get_last_hash_value() + .map(crate::node::hash_str_to_int64) + } + }) + .map(|parent| namespaced_block_hash(namespace_seed.as_ref(), parent)); let num_pages = node.key.atom_len().div_ceil(self.page_size); let mut events = Vec::with_capacity(num_pages); let mut append_event = |page: &[K::Atom], block_hash| { + let block_hash = namespaced_block_hash(namespace_seed.as_ref(), block_hash); events.push(KvCacheEvent::BlockStored { block_hashes: vec![block_hash], parent_block_hash, @@ -3313,6 +3330,7 @@ impl UnifiedTreeCore { medium, cache_salt: cache_salt.clone(), session_id: session_id.clone(), + extra_key: extra_key.clone(), }); parent_block_hash = Some(block_hash); }; @@ -3370,6 +3388,11 @@ impl UnifiedTreeCore { .map(|hash| crate::node::hash_str_to_int64(hash)) .collect() }; + let namespace_seed = self.namespace_seed_(node_id); + let block_hashes: Vec = block_hashes + .into_iter() + .map(|hash| namespaced_block_hash(namespace_seed.as_ref(), hash)) + .collect(); if !block_hashes.is_empty() { self.enqueue_kv_event_(KvCacheEvent::BlockRemoved { block_hashes, @@ -3378,6 +3401,14 @@ impl UnifiedTreeCore { } } + /// The namespaced-hash seed of `node_id`; None in the default format. + fn namespace_seed_(&self, node_id: NodeIdx_) -> Option { + if !self.dynamo_kv_event_format { + return None; + } + kv_event_namespace_seed(&self.arena.node(node_id).namespace) + } + /// Queue the all-cleared marker. pub fn record_all_cleared_event(&mut self) { if self.enable_kv_cache_events { @@ -5750,6 +5781,44 @@ impl StorageMedium { } } +// The dynamo format publishes namespaced hashes as block hashes. If you change +// this tag, all of these hashes change. disaggregation/kv_events.py has a Python copy. +const KV_EVENT_NAMESPACE_SEED_TAG: &[u8] = b"sglang-kv-event-namespace-v1"; + +/// The seed for `namespaced_block_hash`, or None for no namespace. +pub(crate) fn kv_event_namespace_seed(namespace: &KeyNamespace) -> Option { + if *namespace == KeyNamespace::default() { + return None; + } + let mut hasher = Sha256::new(); + hasher.update(KV_EVENT_NAMESPACE_SEED_TAG); + // The presence byte and the length prefix make ("a", "bc") and ("ab", "c") differ. + for part in [namespace.extra_key(), namespace.cache_salt()] { + match part { + None => hasher.update([0u8]), + Some(part) => { + hasher.update([1u8]); + hasher.update((part.len() as u64).to_le_bytes()); + hasher.update(part.as_bytes()); + } + } + } + Some(hasher.finalize().into()) +} + +/// Return the namespaced hash of a published block hash. If there is no namespace, +/// the result is `block_hash`. If not, it is the first 8 bytes of SHA-256(seed, block_hash). +pub(crate) fn namespaced_block_hash(seed: Option<&HashDigest>, block_hash: i64) -> i64 { + let Some(seed) = seed else { + return block_hash; + }; + let mut hasher = Sha256::new(); + hasher.update(seed); + hasher.update(block_hash.to_be_bytes()); + let digest: HashDigest = hasher.finalize().into(); + crate::node::hash_digest_to_int64(&digest) +} + /// A KV placement event; one stored event may carry multiple same-sized pages. #[derive(Clone, PartialEq, Eq, Debug)] pub enum KvCacheEvent { @@ -5761,6 +5830,9 @@ pub enum KvCacheEvent { medium: StorageMedium, cache_salt: Option>, session_id: Option>, + /// Set only in the dynamo format, and not published. Python uses it to + /// find the LoRA adapter name. + extra_key: Option>, }, BlockRemoved { block_hashes: Vec, diff --git a/test/registered/unit/disaggregation/test_kv_events.py b/test/registered/unit/disaggregation/test_kv_events.py index 865e76718a..c209570b0f 100644 --- a/test/registered/unit/disaggregation/test_kv_events.py +++ b/test/registered/unit/disaggregation/test_kv_events.py @@ -26,6 +26,8 @@ from sglang.srt.disaggregation.kv_events import ( NullEventPublisher, StorageMedium, ZmqEventPublisher, + kv_event_namespace_seed, + namespaced_block_hash, resolve_load_pub_range, select_kv_publisher_dp_rank, ) @@ -355,6 +357,20 @@ class TestBlockStoredWireFormat(CustomTestCase): self.assertEqual(len(decoded), 3) +class TestNamespacedBlockHash(CustomTestCase): + def test_values_are_pinned(self): + # Rust has its own copy of this function, and its test uses the same + # values. Thus the two copies must give the same hashes, and the hashes + # must not change. + seed = kv_event_namespace_seed(extra_key="lora-a", cache_salt=None) + self.assertEqual( + namespaced_block_hash(123, namespace_seed=seed), 9220659119954863560 + ) + self.assertEqual( + namespaced_block_hash(-5, namespace_seed=seed), -1810974732006142875 + ) + + class TestReplay(CustomTestCase): def test_descriptor_advertises_replay_port(self): def descriptor(**cfg): diff --git a/test/registered/unit/mem_cache/test_radix_cache_unit.py b/test/registered/unit/mem_cache/test_radix_cache_unit.py index 7b8528201f..d9c5320afb 100644 --- a/test/registered/unit/mem_cache/test_radix_cache_unit.py +++ b/test/registered/unit/mem_cache/test_radix_cache_unit.py @@ -45,7 +45,7 @@ from sglang.srt.mem_cache.base_prefix_cache import ( MatchPrefixParams, ) from sglang.srt.mem_cache.cache_init_params import CacheInitParams -from sglang.srt.mem_cache.events import KVCacheEventRecorder +from sglang.srt.mem_cache.events import KVCacheEventRecorder, LoRANameTable from sglang.srt.mem_cache.radix_cache import RadixCache, RadixKey, TreeNode from sglang.srt.utils import get_device from sglang.test.test_utils import CustomTestCase @@ -134,6 +134,17 @@ class TestKVCacheEventQueue(unittest.TestCase): self.assertEqual(len(queue.take()), 2) +class TestLoRANameTable(unittest.TestCase): + def test_resolves_the_lora_id_suffix_of_extra_key(self): + lora_id = "0123456789abcdef0123456789abcdef" + table = LoRANameTable() + table.register(lora_id=lora_id, lora_name="adapter-a") + + # Req adds lora_id to the end of the extra_key from the caller. + self.assertEqual(table.resolve("tenant-" + lora_id), "adapter-a") + self.assertIsNone(table.resolve("tenant-" + "f" * 32)) + + class TestRadixKey(unittest.TestCase): """Test cases for RadixKey class.""" @@ -783,6 +794,57 @@ class TestRadixCache(CustomTestCase): self.assertEqual(published[0], published[1]) self.assertIsNotNone(published[1][-1][0]) + def test_dynamo_format_separates_lora_blocks_from_base_blocks(self): + """Adapter blocks and base blocks with the same tokens share block hashes in + the default format. The dynamo format gives them different hashes.""" + lora_id = "0123456789abcdef0123456789abcdef" + lora_names = LoRANameTable() + lora_names.register(lora_id=lora_id, lora_name="adapter-a") + tokens = [1, 2, 3, 4] + mock_allocator = unittest.mock.Mock() + mock_allocator.device = torch.device("cpu") + cache = RadixCache.create_simulated( + mock_allocator=mock_allocator, + page_size=2, + enable_kv_cache_events=True, + dynamo_kv_event_format=True, + kv_event_lora_names=lora_names, + ) + for extra_key in (None, lora_id): + cache.insert( + InsertParams( + key=RadixKey(array("q", tokens), extra_key=extra_key), + value=torch.tensor(tokens, dtype=torch.int64), + ) + ) + + base, lora = [e for e in cache.take_events() if isinstance(e, BlockStored)] + self.assertNotEqual(lora.block_hashes, base.block_hashes) + self.assertEqual(lora.lora_name, "adapter-a") + + cache.evict(EvictParams(num_tokens=2 * len(tokens))) + removed = [ + block_hash + for event in cache.take_events() + if isinstance(event, BlockRemoved) + for block_hash in event.block_hashes + ] + self.assertCountEqual(removed, base.block_hashes + lora.block_hashes) + + def test_dynamo_format_parent_link_survives_node_split(self): + cache = RadixCache.create_simulated( + page_size=2, enable_kv_cache_events=True, dynamo_kv_event_format=True + ) + for tokens in ([1, 2, 3, 4], [1, 2, 9, 10]): + cache.insert( + InsertParams( + key=RadixKey(array("q", tokens), extra_key="lora-a"), + value=torch.tensor(tokens, dtype=torch.int64), + ) + ) + first, branch = [e for e in cache.take_events() if isinstance(e, BlockStored)] + self.assertEqual(branch.parent_block_hash, first.block_hashes[0]) + def test_cache_salt_event_hashes_are_preserved_across_node_split(self): cache = RadixCache.create_simulated(page_size=2, enable_kv_cache_events=True) original = RadixKey(array("q", [1, 2, 3, 4]), cache_salt="tenant-a") diff --git a/test/registered/unit/mem_cache/test_rust_tree_core_integration.py b/test/registered/unit/mem_cache/test_rust_tree_core_integration.py index 16885de4da..30675cebc7 100644 --- a/test/registered/unit/mem_cache/test_rust_tree_core_integration.py +++ b/test/registered/unit/mem_cache/test_rust_tree_core_integration.py @@ -20,6 +20,8 @@ from sglang.srt.disaggregation.kv_events import ( BlockRemoved, BlockStored, StorageMedium, + kv_event_namespace_seed, + namespaced_block_hash, ) from sglang.srt.environ import envs from sglang.srt.mem_cache.base_prefix_cache import ( @@ -32,6 +34,7 @@ from sglang.srt.mem_cache.base_prefix_cache import ( TreeLock, ) from sglang.srt.mem_cache.cache_init_params import CacheInitParams +from sglang.srt.mem_cache.events import LoRANameTable from sglang.srt.mem_cache.evict_policy import TLRUStrategy from sglang.srt.mem_cache.hicache_storage import ( PoolHitPolicy, @@ -1674,6 +1677,55 @@ def test_salted_events_match_python_hash_and_metadata_contract(): ] +def test_dynamo_format_events_match_python_hashes_and_name_the_adapter(): + """In the dynamo format, the Rust tree publishes the Python namespaced hashes + and the adapter name.""" + lora_id = "a" * 32 + lora_names = LoRANameTable() + lora_names.register(lora_id=lora_id, lora_name="adapter-a") + core = _tree_core( + enable_kv_cache_events=True, + page_size=2, + dynamo_kv_event_format=True, + kv_event_lora_names=lora_names, + ) + key = RadixKey(array("q", [1, 2, 7, 8]), extra_key="user-" + lora_id) + _pump_insert( + core, + InsertParams(key=key, value=torch.tensor([10, 11, 12, 13], dtype=torch.int64)), + ) + token_only = [ + hash_str_to_int64(value) + for value in mem_cache.get_hash_str(array("q", [1, 2, 7, 8]), None, 2) + ] + seed = kv_event_namespace_seed(extra_key=key.extra_key, cache_salt=None) + namespaced = [namespaced_block_hash(h, namespace_seed=seed) for h in token_only] + assert namespaced != token_only + assert core.take_events() == [ + BlockStored( + block_hashes=namespaced, + parent_block_hash=None, + token_ids=[1, 2, 7, 8], + block_size=2, + lora_id=None, + medium=StorageMedium.GPU, + lora_name="adapter-a", + ) + ] + + tracker = {ComponentType.FULL: 0} + core.evict_device_start(ComponentType.FULL, 4) + candidate = core.evict_device_next_node(ComponentType.FULL, tracker).node_id + assert candidate is not None + evicted = core.evict_device_leaf(candidate, is_write_back=False) + evicted.device_frees.clear() + evicted.host_frees.clear() + core.evict_device_end(ComponentType.FULL) + assert core.take_events() == [ + BlockRemoved(block_hashes=namespaced, medium=StorageMedium.GPU) + ] + + def test_salted_eagle_events_match_the_bigram_hash_contract(): core = _tree_core(enable_kv_cache_events=True, page_size=2, is_eagle=True) raw_tokens = array("q", [1, 2, 3, 4, 5])