[KV events] Publish opaque LoRA block IDs and adapter names (#42817)

Co-authored-by: Kan Wu <kan.wu@radixark.ai>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
Co-authored-by: Shangming Cai <csmthu@gmail.com>
This commit is contained in:
Kan Wu
2026-10-14 22:56:28 +08:00
committed by GitHub
co-authored by Kan Wu Claude Opus 5.5 Shangming Cai
parent ed40c7fc66
commit 010776e94f
18 changed files with 446 additions and 41 deletions
@@ -290,8 +290,8 @@ class BlockStored(KVCacheEvent):
# Session that triggered this store. Attribution only: the blocks may be
# shared with other sessions, and the hash does not depend on it.
session_id: Optional[str] = None
# The LoRA adapter name of the request that stored these blocks.
# Set only in the dynamo format.
# Public adapter name for router lookup. Published hashes are opaque IDs
# scoped to the internal cache namespace, including the adapter load.
lora_name: Optional[str] = None
+4
View File
@@ -1173,6 +1173,8 @@ class TokenizedGenerateReqInput(BaseReq, kw_only=True):
# Shape of output_token_sampling_logprobs for each output token. This is a
# defaulted tail field so older IPC senders decode as selected mode.
sampling_logprobs_mode: SamplingLogprobsMode = "selected"
# LoRA adapter name; KV events namespace this request's blocks by it.
lora_name: Optional[str] = None
def wrap_pickle_fields(self):
self.time_stats = wrap_as_pickle(self.time_stats)
@@ -1537,6 +1539,8 @@ class TokenizedEmbeddingReqInput(BaseReq, kw_only=True):
# For observability
# Pickled Optional[Union[APIServerReqTimeStats, DPControllerReqTimeStats]]
time_stats: Optional[PickleWrapper] = None
# LoRA adapter name; KV events namespace this request's blocks by it.
lora_name: Optional[str] = None
def wrap_pickle_fields(self):
self.time_stats = wrap_as_pickle(self.time_stats)
+15 -2
View File
@@ -298,7 +298,7 @@ from sglang.srt.mem_cache.common import (
discard_kv_cache_backup,
release_kv_cache,
)
from sglang.srt.mem_cache.events import LoRANameTable
from sglang.srt.mem_cache.events import KvEventLoraNames
from sglang.srt.model_executor.forward_batch_info import PPProxyTensors
from sglang.srt.model_executor.runner_utils.pool import prewarm_graph_pool_borrow
from sglang.srt.model_loader.utils import get_resolved_model_impl
@@ -2451,7 +2451,9 @@ class Scheduler(
def init_kv_event_lora_names(self) -> None:
"""Init the table that maps lora_id to adapter name for KV events."""
self.kv_event_lora_names = LoRANameTable.from_lora_refs(get_lora().lora_paths)
self.kv_event_lora_names = KvEventLoraNames.from_lora_refs(
get_lora().lora_paths
)
def init_kv_events_publisher(self) -> None:
self.kv_events_publisher = SchedulerKvEventsPublisher(
@@ -2778,6 +2780,15 @@ class Scheduler(
mm.mrope_positions = mrope_positions
mm.mrope_position_delta = mrope_position_delta
def _record_kv_event_lora_name(self, req: Req, lora_name: Optional[str]) -> None:
# Keyed by the final extra_key, so it must run after elastic namespacing.
# Only a cache whose events are taken prunes the table.
if (
self.kv_events_publisher.enable_kv_cache_events
and self.tree_cache.supports_prefix_sharing()
):
self.kv_event_lora_names.register(req, lora_name)
def _maybe_namespace_elastic_radix_cache(self, req: Req) -> None:
if (
get_exec().moe.elastic_ep_backend is None
@@ -2990,6 +3001,7 @@ class Scheduler(
if recv_req.pp_prefetch_ticketed is True:
self.tree_cache.bind_prefetch_ticket(req.rid)
self._maybe_namespace_elastic_radix_cache(req)
self._record_kv_event_lora_name(req, recv_req.lora_name)
if mm_input_error is not None:
req.set_finish_with_abort(
@@ -3566,6 +3578,7 @@ class Scheduler(
)
req.tokenizer = self.tokenizer
self._maybe_namespace_elastic_radix_cache(req)
self._record_kv_event_lora_name(req, recv_req.lora_name)
if mm_input_error is not None:
req.set_finish_with_abort(
@@ -1602,6 +1602,7 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin):
bootstrap_port=obj.bootstrap_port,
bootstrap_room=bootstrap_room,
lora_id=obj.lora_id,
lora_name=obj.lora_path,
input_embeds=input_embeds,
positional_embed_overrides=obj.positional_embed_overrides,
session_id=obj.session_id,
@@ -1649,6 +1650,7 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin):
priority=obj.priority,
dimensions=obj.dimensions,
lora_id=obj.lora_id,
lora_name=obj.lora_path,
http_worker_ipc=obj.http_worker_ipc,
return_pooled_hidden_states=obj.return_pooled_hidden_states,
multi_item_delimiter_indices=obj.multi_item_delimiter_indices,
@@ -5,9 +5,10 @@ from typing import TYPE_CHECKING, Any, Optional
import torch
from sglang.srt.mem_cache.events import KvEventLoraNames
if TYPE_CHECKING:
from sglang.srt.mem_cache.allocator import BaseTokenToKVPoolAllocator
from sglang.srt.mem_cache.events import LoRANameTable
from sglang.srt.mem_cache.memory_pool import ReqToTokenPool
from sglang.srt.mem_cache.unified_cache.components import ComponentType
from sglang.srt.mem_cache.unified_cache.components.base import (
@@ -35,8 +36,10 @@ class CacheInitParams:
enable_metrics: bool = False
enable_kv_cache_events: bool = False
dynamo_kv_event_format: bool = False
# The scheduler holds the same table and adds each adapter when it loads.
kv_event_lora_names: Optional[LoRANameTable] = None
# LoRA names for KV events; the scheduler registers requests in it.
kv_event_lora_names: KvEventLoraNames = dataclasses.field(
default_factory=KvEventLoraNames
)
enable_session_radix_cache: bool = False
enable_mamba_extra_buffer: bool = False
+59 -5
View File
@@ -65,12 +65,57 @@ class LoRANameTable:
def register(self, *, lora_id: str, lora_name: str) -> None:
self._names[lora_id] = lora_name
def get(self, extra_key: Optional[str]) -> Optional[str]:
return self.resolve(extra_key)
def resolve(self, extra_key: Optional[str]) -> Optional[str]:
if not extra_key:
return None
return self._names.get(extra_key[-_LORA_ID_LEN:])
class KvEventLoraNames(LoRANameTable):
"""Public names of selected LoRA namespaces, kept for the publisher lifetime.
Write-back can publish a host store after the last GPU removal and after
its request dies, so request or published-block counts cannot bound this.
"""
def __init__(self):
# Adapter-id registrations keep the existing Dynamo load-time lookup.
self._adapter_names: dict[str, str] = {}
self._names: dict[str, str] = {}
def register(
self,
req: Any = None,
lora_name: Optional[str] = None,
*,
lora_id: Optional[str] = None,
) -> None:
"""Name ``req``'s namespace; ``req`` needs ``extra_key`` and ``lora_id``."""
if lora_id is not None:
self._adapter_names[lora_id] = lora_name
return
if req.lora_id is None:
return
# Older IPC senders omit the optional name; admission still resolves an ID.
lora_name = lora_name or self._adapter_names.get(req.lora_id)
if not lora_name:
return
self._names[req.extra_key] = lora_name
def get(self, extra_key: Optional[str]) -> Optional[str]:
return self._names.get(extra_key)
def resolve(self, extra_key: Optional[str]) -> Optional[str]:
if not extra_key:
return None
return self._names.get(extra_key) or self._adapter_names.get(
extra_key[-_LORA_ID_LEN:]
)
class KVCacheEventRecorder:
"""Collects KV placement events for one cache.
@@ -84,12 +129,12 @@ class KVCacheEventRecorder:
enabled: bool,
page_size: int,
dynamo_format: bool = False,
lora_names: Optional[LoRANameTable] = None,
lora_names: Optional[KvEventLoraNames] = None,
):
self.enabled = enabled
self.page_size = page_size
self.dynamo_format = dynamo_format
self.lora_names = lora_names if lora_names is not None else LoRANameTable()
self.lora_names = lora_names if lora_names is not None else KvEventLoraNames()
self._queue: list = []
def enqueue(self, event) -> None:
@@ -114,6 +159,7 @@ class KVCacheEventRecorder:
and tail.block_size == event.block_size
and tail.cache_salt == event.cache_salt
and tail.session_id == event.session_id
and tail.lora_name == event.lora_name
and tail.block_hashes
and event.parent_block_hash == tail.block_hashes[-1]
):
@@ -151,9 +197,15 @@ class KVCacheEventRecorder:
return hash_str_to_int64(parent_hash_values[-1])
def _namespace_seed(self, node: Any) -> Optional[bytes]:
# The default format publishes the hashes unchanged.
if not self.dynamo_format:
return None
return kv_event_namespace_seed(
extra_key=(
node.key.extra_key
if self.lora_names.get(node.key.extra_key) is not None
else None
),
cache_salt=None,
)
return kv_event_namespace_seed(
extra_key=node.key.extra_key, cache_salt=node.key.cache_salt
)
@@ -177,7 +229,9 @@ class KVCacheEventRecorder:
parent_block_hash, namespace_seed=namespace_seed
)
lora_name = (
self.lora_names.resolve(node.key.extra_key) if self.dynamo_format else None
self.lora_names.resolve(node.key.extra_key)
if self.dynamo_format
else self.lora_names.get(node.key.extra_key)
)
page_index = 0
@@ -40,6 +40,7 @@ from sglang.srt.managers.mm_schedule import init_mm_embedding_cache
from sglang.srt.mem_cache.base_swa_memory_pool import BaseSWAKVPool
from sglang.srt.mem_cache.cache_init_params import CacheInitParams
from sglang.srt.mem_cache.deepseek_v4_memory_pool import DeepSeekV4TokenToKVPool
from sglang.srt.mem_cache.events import KvEventLoraNames
from sglang.srt.mem_cache.hicache_auto_size import auto_size_hicache
from sglang.srt.mem_cache.memory_pool import HybridLinearKVPool, MHATokenToKVPool
from sglang.srt.mem_cache.pool_host.base import _WRITE_BACK_STAGING_PAGE_CHUNK
@@ -61,7 +62,6 @@ from sglang.srt.utils import ceil_align, is_hip
if TYPE_CHECKING:
from sglang.srt.configs.model_config import ModelConfig
from sglang.srt.managers.tp_worker import BaseTpWorker
from sglang.srt.mem_cache.events import LoRANameTable
from sglang.srt.server_args import ServerArgs
from sglang.srt.speculative.base_spec_worker import HiCacheDraftPlan
from sglang.srt.speculative.spec_info import SpeculativeAlgorithm
@@ -257,7 +257,7 @@ def build_kv_cache(
enable_hierarchical_cache: bool,
hicache_draft_plan: Optional[HiCacheDraftPlan] = None,
dynamo_kv_event_format: bool = False,
kv_event_lora_names: Optional[LoRANameTable] = None,
kv_event_lora_names: Optional[KvEventLoraNames] = None,
) -> KVCacheBuildResult:
parallel = get_parallel()
sliding_window_size: Optional[int] = None
+1 -1
View File
@@ -71,7 +71,7 @@ class RadixKey:
):
# token ids sequence (raw ints in both modes)
self.token_ids = token_ids
# Namespaces the tree and storage; omitted from KV events.
# Namespaces the tree and storage; KV events see only its LoRA name.
self.extra_key = extra_key
# Namespaces the tree, storage and KV events.
self.cache_salt = cache_salt or None
@@ -14,6 +14,8 @@ from sglang.srt.disaggregation.kv_events import (
BlockRemoved,
BlockStored,
StorageMedium,
kv_event_namespace_seed,
namespaced_block_hash,
)
from sglang.srt.environ import envs
from sglang.srt.mem_cache.allocator.swa import is_swa_req_ring
@@ -26,7 +28,7 @@ from sglang.srt.mem_cache.base_prefix_cache import (
MatchPrefixParams,
MatchResult,
)
from sglang.srt.mem_cache.events import LoRANameTable
from sglang.srt.mem_cache.events import KvEventLoraNames
from sglang.srt.mem_cache.hicache_storage import PoolHitPolicy, PoolName, PoolTransfer
from sglang.srt.mem_cache.radix_cache import RadixKey
from sglang.srt.mem_cache.rust_tree_core.extension import bindings
@@ -60,7 +62,9 @@ from sglang.srt.mem_cache.unified_cache.unified_tree_core_interface import (
TreePathNodeInfo,
UnifiedTreeCoreInterface,
)
from sglang.srt.mem_cache.utils import get_eviction_strategy
from sglang.srt.mem_cache.utils import (
get_eviction_strategy,
)
from sglang.srt.runtime_context import get_exec, mamba_cache_chunk_size
from sglang.srt.utils import assert_int64_array
@@ -97,23 +101,54 @@ def _radix_key_buffer(key: RadixKey) -> array:
return token_ids
def _kv_event_from_tagged(event: tuple, lora_names: LoRANameTable):
"""Build the Python KV cache event for one of the binding's tagged tuples."""
def _kv_event_from_tagged(
event: tuple, lora_names: KvEventLoraNames, dynamo_format: bool = False
):
"""Build the Python KV cache event for one of the binding's tagged tuples.
Rust hashes are LoRA-agnostic; the trailing extra_key selects the LoRA
namespace, applied here as in KVCacheEventRecorder.
"""
tag = event[0]
if tag == "block_stored":
lora_name = (
lora_names.resolve(event[8]) if dynamo_format else lora_names.get(event[8])
)
seed = (
kv_event_namespace_seed(extra_key=event[8], cache_salt=None)
if not dynamo_format and lora_name is not None
else None
)
parent = event[2]
return BlockStored(
block_hashes=event[1],
parent_block_hash=event[2],
block_hashes=[
namespaced_block_hash(h, namespace_seed=seed) for h in event[1]
],
parent_block_hash=(
None
if parent is None
else namespaced_block_hash(parent, namespace_seed=seed)
),
token_ids=event[3],
block_size=event[4],
lora_id=None,
medium=StorageMedium(event[5]),
cache_salt=event[6],
session_id=event[7],
lora_name=lora_names.resolve(event[8]),
lora_name=lora_name,
)
if tag == "block_removed":
return BlockRemoved(block_hashes=event[1], medium=StorageMedium(event[2]))
seed = (
kv_event_namespace_seed(extra_key=event[3], cache_salt=None)
if not dynamo_format and lora_names.get(event[3]) is not None
else None
)
return BlockRemoved(
block_hashes=[
namespaced_block_hash(h, namespace_seed=seed) for h in event[1]
],
medium=StorageMedium(event[2]),
)
if tag == "all_blocks_cleared":
return AllBlocksCleared()
raise ValueError(f"unknown kv event tag: {tag}")
@@ -305,20 +340,26 @@ class _RustKVCacheEventRecorder:
"""Expose the Rust event queue through the Python recorder interface."""
def __init__(
self, binding, enabled: bool, lora_names: Optional[LoRANameTable] = None
self,
binding,
enabled: bool,
lora_names: KvEventLoraNames,
dynamo_format: bool = False,
):
self._binding = binding
self.enabled = enabled
self.lora_names = lora_names if lora_names is not None else LoRANameTable()
self._lora_names = lora_names
self._dynamo_format = dynamo_format
def record_all_cleared(self) -> None:
self._binding.record_all_cleared_event()
def take(self) -> list:
return [
_kv_event_from_tagged(event, self.lora_names)
events = [
_kv_event_from_tagged(event, self._lora_names, self._dynamo_format)
for event in self._binding.take_events()
]
return events
class RustUnifiedTreeCore(UnifiedTreeCoreInterface):
@@ -445,8 +486,9 @@ class RustUnifiedTreeCore(UnifiedTreeCoreInterface):
)
self.kv_events = _RustKVCacheEventRecorder(
self._binding,
params.enable_kv_cache_events,
enabled=params.enable_kv_cache_events,
lora_names=params.kv_event_lora_names,
dynamo_format=params.dynamo_kv_event_format,
)
# The default-root empty result, prebuilt once from the binding.
self._empty_match_result = _match_result_from_binding(