[KV Events] Publish LoRA adapter names and namespaced block hashes (#42455)

Signed-off-by: Banruo Liu <lichangxinlivingshade@gmail.com>
Co-authored-by: Shangming Cai <csmthu@gmail.com>
This commit is contained in:
Banruo Liu
2026-10-11 23:23:30 +08:00
committed by GitHub
co-authored by Shangming Cai
parent 97bfa5cc53
commit d62b9c2480
15 changed files with 546 additions and 26 deletions
+55 -1
View File
@@ -19,6 +19,7 @@ KV caching events
import atexit
import enum
import hashlib
import logging
import os
import queue
@@ -28,7 +29,7 @@ from abc import ABC, abstractmethod
from collections import deque
from itertools import count
from queue import Queue
from typing import Any, Callable, Optional, Union
from typing import Any, Callable, Literal, Optional, Union
import msgspec
import zmq
@@ -289,6 +290,9 @@ class BlockStored(KVCacheEvent):
# Session that triggered this store. Attribution only: the blocks may be
# shared with other sessions, and the hash does not depend on it.
session_id: Optional[str] = None
# The LoRA adapter name of the request that stored these blocks.
# Set only in the dynamo format.
lora_name: Optional[str] = None
class BlockRemoved(KVCacheEvent):
@@ -300,6 +304,42 @@ class AllBlocksCleared(KVCacheEvent):
pass
# The dynamo format publishes namespaced hashes as block hashes. If you change
# this tag, all of these hashes change. unified_tree_core.rs has a Rust copy.
_NAMESPACE_SEED_TAG = b"sglang-kv-event-namespace-v1"
def kv_event_namespace_seed(
*, extra_key: Optional[str], cache_salt: Optional[str]
) -> Optional[bytes]:
"""Return the seed for namespaced_block_hash, or None for no namespace."""
if extra_key is None and cache_salt is None:
return None
digest = hashlib.sha256(_NAMESPACE_SEED_TAG)
# The presence byte and the length prefix make ("a", "bc") and ("ab", "c") differ.
for part in (extra_key, cache_salt):
if part is None:
digest.update(b"\x00")
continue
encoded = part.encode("utf-8")
digest.update(b"\x01" + len(encoded).to_bytes(8, "little") + encoded)
return digest.digest()
def namespaced_block_hash(block_hash: int, *, namespace_seed: Optional[bytes]) -> int:
"""Return the namespaced hash of a published block hash.
If namespace_seed is None, the result is ``block_hash``. If not, the result is
the first 8 bytes of SHA-256(seed, block_hash as 8 big-endian bytes), as int64.
"""
if namespace_seed is None:
return block_hash
digest = hashlib.sha256(
namespace_seed + block_hash.to_bytes(8, "big", signed=True)
).digest()
return int.from_bytes(digest[:8], "big", signed=True)
class KVEventBatch(EventBatch):
events: list[Union[BlockStored, BlockRemoved, AllBlocksCleared]]
@@ -680,12 +720,24 @@ class KVEventsConfig(BaseModel):
this topic to receive events.
"""
format: Literal["default", "dynamo"] = "default"
"""The event format. "default" is the SGLang format. "dynamo" is the format
the Dynamo KV router reads: block hashes also depend on extra_key and
cache_salt (see ``namespaced_block_hash``), and BlockStored has lora_name.
"""
@classmethod
def from_cli(cls, cli_value: str) -> "KVEventsConfig":
"""Parse the CLI value for the event publisher config."""
return KVEventsConfig.model_validate_json(cli_value)
def uses_dynamo_format(kv_events_config: Optional[str]) -> bool:
if not kv_events_config:
return False
return KVEventsConfig.from_cli(kv_events_config).format == "dynamo"
class EventPublisherFactory:
_registry: dict[str, Callable[..., EventPublisher]] = {
"null": NullEventPublisher,
@@ -705,6 +757,8 @@ class EventPublisherFactory:
return NullEventPublisher()
config = KVEventsConfig.from_cli(config)
config_dict = config.model_dump()
# The cache event recorder uses the format. The publisher does not accept it.
config_dict.pop("format")
kind = config_dict.pop("publisher", "null")
try:
+19
View File
@@ -87,6 +87,7 @@ from sglang.srt.disaggregation.decode_kvcache_offload_manager import (
DecodeKVCacheOffloadManager,
)
from sglang.srt.disaggregation.encoder.receiver import create_mm_receiver
from sglang.srt.disaggregation.kv_events import uses_dynamo_format
from sglang.srt.disaggregation.prefill import (
PrefillBootstrapQueue,
SchedulerDisaggregationPrefillMixin,
@@ -297,6 +298,7 @@ from sglang.srt.mem_cache.common import (
discard_kv_cache_backup,
release_kv_cache,
)
from sglang.srt.mem_cache.events import LoRANameTable
from sglang.srt.model_executor.forward_batch_info import PPProxyTensors
from sglang.srt.model_executor.runner_utils.pool import prewarm_graph_pool_borrow
from sglang.srt.model_loader.utils import get_resolved_model_impl
@@ -564,6 +566,7 @@ class Scheduler(
time.sleep(t)
# Init cache and memory pool
self.init_kv_event_lora_names()
result = kv_cache_builder.build_kv_cache(
server_args=self.server_args,
model_config=self.model_config,
@@ -577,6 +580,10 @@ class Scheduler(
and get_parallel().attn_tp_rank == 0
and get_parallel().attn_cp_rank == 0
),
dynamo_kv_event_format=uses_dynamo_format(
get_observability().kv_events_config
),
kv_event_lora_names=self.kv_event_lora_names,
enable_hierarchical_cache=self.enable_hierarchical_cache,
hicache_draft_plan=(
self.draft_worker.hicache_draft_plan
@@ -2429,6 +2436,10 @@ class Scheduler(
groups += [self.pp_group]
rank_consensus_checker.configure(groups)
def init_kv_event_lora_names(self) -> None:
"""Init the table that maps lora_id to adapter name for KV events."""
self.kv_event_lora_names = LoRANameTable.from_lora_refs(get_lora().lora_paths)
def init_kv_events_publisher(self) -> None:
self.kv_events_publisher = SchedulerKvEventsPublisher(
kv_events_config=get_observability().kv_events_config,
@@ -5768,6 +5779,10 @@ class Scheduler(
"""In-place loading a new lora adapter from disk or huggingface."""
result = self.tp_worker.load_lora_adapter(recv_req)
if result.success:
self.kv_event_lora_names.register(
lora_id=recv_req.lora_id, lora_name=recv_req.lora_name
)
return result
def load_lora_adapter_from_tensors(
@@ -5776,6 +5791,10 @@ class Scheduler(
"""In-place loading a new lora adapter from serialized tensors."""
result = self.tp_worker.load_lora_adapter_from_tensors(recv_req)
if result.success:
self.kv_event_lora_names.register(
lora_id=recv_req.lora_id, lora_name=recv_req.lora_name
)
return result
def unload_lora_adapter(
@@ -7,6 +7,7 @@ import torch
if TYPE_CHECKING:
from sglang.srt.mem_cache.allocator import BaseTokenToKVPoolAllocator
from sglang.srt.mem_cache.events import LoRANameTable
from sglang.srt.mem_cache.memory_pool import ReqToTokenPool
from sglang.srt.mem_cache.unified_cache.components import ComponentType
from sglang.srt.mem_cache.unified_cache.components.base import (
@@ -33,6 +34,9 @@ class CacheInitParams:
enable_metrics: bool = False
enable_kv_cache_events: bool = False
dynamo_kv_event_format: bool = False
# The scheduler holds the same table and adds each adapter when it loads.
kv_event_lora_names: Optional[LoRANameTable] = None
enable_session_radix_cache: bool = False
enable_mamba_extra_buffer: bool = False
+77 -4
View File
@@ -18,13 +18,17 @@ consumed by KV-aware routers (e.g. dynamo). A cache holds one recorder and calls
it; the recorder owns the queue and needs nothing back from its owner.
"""
from typing import Any, Optional
from __future__ import annotations
from typing import TYPE_CHECKING, Any, Iterable, Optional
from sglang.srt.disaggregation.kv_events import (
AllBlocksCleared,
BlockRemoved,
BlockStored,
StorageMedium,
kv_event_namespace_seed,
namespaced_block_hash,
)
from sglang.srt.mem_cache.utils import (
compute_node_event_hash_values,
@@ -32,6 +36,40 @@ from sglang.srt.mem_cache.utils import (
hash_str_to_int64,
)
if TYPE_CHECKING:
from sglang.srt.lora.lora_registry import LoRARef
# Length of LoRARef.lora_id, a uuid4 or uuid5 hex string. Req adds the id to the
# end of extra_key, so the last 32 characters of extra_key are the id.
_LORA_ID_LEN = 32
class LoRANameTable:
"""Find the LoRA adapter name from the lora_id at the end of extra_key.
Req adds lora_id to the end of extra_key, so a tree node has only the id.
The table keeps all entries. A lora_id is not used again, and nodes of an
unloaded adapter can publish stores until eviction.
"""
def __init__(self):
self._names: dict[str, str] = {}
@classmethod
def from_lora_refs(cls, lora_refs: Optional[Iterable[LoRARef]]) -> LoRANameTable:
table = cls()
for lora_ref in lora_refs or ():
table.register(lora_id=lora_ref.lora_id, lora_name=lora_ref.lora_name)
return table
def register(self, *, lora_id: str, lora_name: str) -> None:
self._names[lora_id] = lora_name
def resolve(self, extra_key: Optional[str]) -> Optional[str]:
if not extra_key:
return None
return self._names.get(extra_key[-_LORA_ID_LEN:])
class KVCacheEventRecorder:
"""Collects KV placement events for one cache.
@@ -40,9 +78,18 @@ class KVCacheEventRecorder:
empty list, so callers never have to guard.
"""
def __init__(self, *, enabled: bool, page_size: int):
def __init__(
self,
*,
enabled: bool,
page_size: int,
dynamo_format: bool = False,
lora_names: Optional[LoRANameTable] = None,
):
self.enabled = enabled
self.page_size = page_size
self.dynamo_format = dynamo_format
self.lora_names = lora_names if lora_names is not None else LoRANameTable()
self._queue: list = []
def enqueue(self, event) -> None:
@@ -103,6 +150,14 @@ class KVCacheEventRecorder:
return None
return hash_str_to_int64(parent_hash_values[-1])
def _namespace_seed(self, node: Any) -> Optional[bytes]:
# The default format publishes the hashes unchanged.
if not self.dynamo_format:
return None
return kv_event_namespace_seed(
extra_key=node.key.extra_key, cache_salt=node.key.cache_salt
)
def record_store(
self, node: Any, medium=None, *, session_id: Optional[str] = None
) -> None:
@@ -115,7 +170,15 @@ class KVCacheEventRecorder:
medium = StorageMedium.GPU
event_hash_values = self._node_event_hash_values(node)
namespace_seed = self._namespace_seed(node)
parent_block_hash = self._parent_block_hash(node)
if parent_block_hash is not None:
parent_block_hash = namespaced_block_hash(
parent_block_hash, namespace_seed=namespace_seed
)
lora_name = (
self.lora_names.resolve(node.key.extra_key) if self.dynamo_format else None
)
page_index = 0
logical_len = len(node.key)
@@ -131,7 +194,10 @@ class KVCacheEventRecorder:
else:
page_tokens = list(raw[start:end])
block_hash = hash_str_to_int64(event_hash_values[page_index])
block_hash = namespaced_block_hash(
hash_str_to_int64(event_hash_values[page_index]),
namespace_seed=namespace_seed,
)
self.enqueue(
BlockStored(
@@ -143,6 +209,7 @@ class KVCacheEventRecorder:
medium=medium,
cache_salt=node.key.cache_salt,
session_id=session_id,
lora_name=lora_name,
)
)
@@ -160,6 +227,7 @@ class KVCacheEventRecorder:
# Hash values must match what was stored.
event_hash_values = self._node_event_hash_values(node)
namespace_seed = self._namespace_seed(node)
block_hashes = []
logical_len = len(node.key)
@@ -169,7 +237,12 @@ class KVCacheEventRecorder:
if end <= start:
continue
block_hashes.append(hash_str_to_int64(event_hash_values[page_index]))
block_hashes.append(
namespaced_block_hash(
hash_str_to_int64(event_hash_values[page_index]),
namespace_seed=namespace_seed,
)
)
page_index += 1
if block_hashes:
@@ -61,6 +61,7 @@ from sglang.srt.utils import ceil_align, is_hip
if TYPE_CHECKING:
from sglang.srt.configs.model_config import ModelConfig
from sglang.srt.managers.tp_worker import BaseTpWorker
from sglang.srt.mem_cache.events import LoRANameTable
from sglang.srt.server_args import ServerArgs
from sglang.srt.speculative.base_spec_worker import HiCacheDraftPlan
from sglang.srt.speculative.spec_info import SpeculativeAlgorithm
@@ -255,6 +256,8 @@ def build_kv_cache(
enable_kv_cache_events: bool,
enable_hierarchical_cache: bool,
hicache_draft_plan: Optional[HiCacheDraftPlan] = None,
dynamo_kv_event_format: bool = False,
kv_event_lora_names: Optional[LoRANameTable] = None,
) -> KVCacheBuildResult:
parallel = get_parallel()
sliding_window_size: Optional[int] = None
@@ -346,6 +349,8 @@ def build_kv_cache(
eviction_policy_config=get_memory().radix_eviction_policy_config,
enable_metrics=enable_metrics,
enable_kv_cache_events=enable_kv_cache_events,
dynamo_kv_event_format=dynamo_kv_event_format,
kv_event_lora_names=kv_event_lora_names,
enable_session_radix_cache=get_memory().enable_session_radix_cache,
enable_mamba_extra_buffer=get_exec().mamba.enable_mamba_extra_buffer,
enable_mamba_extra_buffer_lazy=get_exec().mamba.enable_mamba_extra_buffer_lazy,
+7 -1
View File
@@ -45,7 +45,7 @@ from sglang.srt.mem_cache.base_prefix_cache import (
MatchPrefixParams,
MatchResult,
)
from sglang.srt.mem_cache.events import KVCacheEventRecorder
from sglang.srt.mem_cache.events import KVCacheEventRecorder, LoRANameTable
from sglang.srt.mem_cache.utils import (
get_eviction_strategy,
split_node_hash_value,
@@ -292,6 +292,8 @@ class RadixCache(BasePrefixCache):
self.kv_events = KVCacheEventRecorder(
enabled=params.enable_kv_cache_events and not self.disable,
page_size=self.page_size,
dynamo_format=params.dynamo_kv_event_format,
lora_names=params.kv_event_lora_names,
)
if params.enable_metrics:
@@ -321,6 +323,8 @@ class RadixCache(BasePrefixCache):
mock_allocator: Optional[Any] = None,
page_size: int = 1,
enable_kv_cache_events: bool = False,
dynamo_kv_event_format: bool = False,
kv_event_lora_names: Optional[LoRANameTable] = None,
) -> RadixCache:
"""Init a radix cache without memory pools for simulation purpose."""
params = CacheInitParams(
@@ -329,6 +333,8 @@ class RadixCache(BasePrefixCache):
token_to_kv_pool_allocator=mock_allocator,
page_size=page_size,
enable_kv_cache_events=enable_kv_cache_events,
dynamo_kv_event_format=dynamo_kv_event_format,
kv_event_lora_names=kv_event_lora_names,
)
return RadixCache(params)
@@ -26,6 +26,7 @@ from sglang.srt.mem_cache.base_prefix_cache import (
MatchPrefixParams,
MatchResult,
)
from sglang.srt.mem_cache.events import LoRANameTable
from sglang.srt.mem_cache.hicache_storage import PoolHitPolicy, PoolName, PoolTransfer
from sglang.srt.mem_cache.radix_cache import RadixKey
from sglang.srt.mem_cache.rust_tree_core.extension import bindings
@@ -95,7 +96,7 @@ def _radix_key_buffer(key: RadixKey) -> array:
return token_ids
def _kv_event_from_tagged(event: tuple):
def _kv_event_from_tagged(event: tuple, lora_names: LoRANameTable):
"""Build the Python KV cache event for one of the binding's tagged tuples."""
tag = event[0]
if tag == "block_stored":
@@ -108,6 +109,7 @@ def _kv_event_from_tagged(event: tuple):
medium=StorageMedium(event[5]),
cache_salt=event[6],
session_id=event[7],
lora_name=lora_names.resolve(event[8]),
)
if tag == "block_removed":
return BlockRemoved(block_hashes=event[1], medium=StorageMedium(event[2]))
@@ -301,15 +303,21 @@ def _fill_evict_result(binding_result, result):
class _RustKVCacheEventRecorder:
"""Expose the Rust event queue through the Python recorder interface."""
def __init__(self, binding, enabled: bool):
def __init__(
self, binding, enabled: bool, lora_names: Optional[LoRANameTable] = None
):
self._binding = binding
self.enabled = enabled
self.lora_names = lora_names if lora_names is not None else LoRANameTable()
def record_all_cleared(self) -> None:
self._binding.record_all_cleared_event()
def take(self) -> list:
return [_kv_event_from_tagged(event) for event in self._binding.take_events()]
return [
_kv_event_from_tagged(event, self.lora_names)
for event in self._binding.take_events()
]
class RustUnifiedTreeCore(UnifiedTreeCoreInterface):
@@ -422,6 +430,7 @@ class RustUnifiedTreeCore(UnifiedTreeCoreInterface):
swa_sliding_window_size=params.sliding_window_size,
swa_req_ring=is_swa_req_ring(self._allocator),
enable_kv_cache_events=params.enable_kv_cache_events,
dynamo_kv_event_format=params.dynamo_kv_event_format,
mamba_cache_chunk_size=(
mamba_cache_chunk_size() if has_mamba else None
),
@@ -434,7 +443,9 @@ class RustUnifiedTreeCore(UnifiedTreeCoreInterface):
[int(component) for component in self.tree_components],
)
self.kv_events = _RustKVCacheEventRecorder(
self._binding, params.enable_kv_cache_events
self._binding,
params.enable_kv_cache_events,
lora_names=params.kv_event_lora_names,
)
# The default-root empty result, prebuilt once from the binding.
self._empty_match_result = _match_result_from_binding(
@@ -460,7 +460,10 @@ class UnifiedTreeCore(UnifiedTreeCoreInterface):
)
self.kv_events = KVCacheEventRecorder(
enabled=params.enable_kv_cache_events, page_size=self.page_size
enabled=params.enable_kv_cache_events,
page_size=self.page_size,
dynamo_format=params.dynamo_kv_event_format,
lora_names=params.kv_event_lora_names,
)
self.reset()