[KV events] Publish opaque LoRA block IDs and adapter names (#42817)
Co-authored-by: Kan Wu <kan.wu@radixark.ai> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: Shangming Cai <csmthu@gmail.com>
This commit is contained in:
co-authored by
Kan Wu
Claude Opus 5.5
Shangming Cai
parent
ed40c7fc66
commit
010776e94f
@@ -290,8 +290,8 @@ class BlockStored(KVCacheEvent):
|
||||
# Session that triggered this store. Attribution only: the blocks may be
|
||||
# shared with other sessions, and the hash does not depend on it.
|
||||
session_id: Optional[str] = None
|
||||
# The LoRA adapter name of the request that stored these blocks.
|
||||
# Set only in the dynamo format.
|
||||
# Public adapter name for router lookup. Published hashes are opaque IDs
|
||||
# scoped to the internal cache namespace, including the adapter load.
|
||||
lora_name: Optional[str] = None
|
||||
|
||||
|
||||
|
||||
@@ -1173,6 +1173,8 @@ class TokenizedGenerateReqInput(BaseReq, kw_only=True):
|
||||
# Shape of output_token_sampling_logprobs for each output token. This is a
|
||||
# defaulted tail field so older IPC senders decode as selected mode.
|
||||
sampling_logprobs_mode: SamplingLogprobsMode = "selected"
|
||||
# LoRA adapter name; KV events namespace this request's blocks by it.
|
||||
lora_name: Optional[str] = None
|
||||
|
||||
def wrap_pickle_fields(self):
|
||||
self.time_stats = wrap_as_pickle(self.time_stats)
|
||||
@@ -1537,6 +1539,8 @@ class TokenizedEmbeddingReqInput(BaseReq, kw_only=True):
|
||||
# For observability
|
||||
# Pickled Optional[Union[APIServerReqTimeStats, DPControllerReqTimeStats]]
|
||||
time_stats: Optional[PickleWrapper] = None
|
||||
# LoRA adapter name; KV events namespace this request's blocks by it.
|
||||
lora_name: Optional[str] = None
|
||||
|
||||
def wrap_pickle_fields(self):
|
||||
self.time_stats = wrap_as_pickle(self.time_stats)
|
||||
|
||||
@@ -298,7 +298,7 @@ from sglang.srt.mem_cache.common import (
|
||||
discard_kv_cache_backup,
|
||||
release_kv_cache,
|
||||
)
|
||||
from sglang.srt.mem_cache.events import LoRANameTable
|
||||
from sglang.srt.mem_cache.events import KvEventLoraNames
|
||||
from sglang.srt.model_executor.forward_batch_info import PPProxyTensors
|
||||
from sglang.srt.model_executor.runner_utils.pool import prewarm_graph_pool_borrow
|
||||
from sglang.srt.model_loader.utils import get_resolved_model_impl
|
||||
@@ -2451,7 +2451,9 @@ class Scheduler(
|
||||
|
||||
def init_kv_event_lora_names(self) -> None:
|
||||
"""Init the table that maps lora_id to adapter name for KV events."""
|
||||
self.kv_event_lora_names = LoRANameTable.from_lora_refs(get_lora().lora_paths)
|
||||
self.kv_event_lora_names = KvEventLoraNames.from_lora_refs(
|
||||
get_lora().lora_paths
|
||||
)
|
||||
|
||||
def init_kv_events_publisher(self) -> None:
|
||||
self.kv_events_publisher = SchedulerKvEventsPublisher(
|
||||
@@ -2778,6 +2780,15 @@ class Scheduler(
|
||||
mm.mrope_positions = mrope_positions
|
||||
mm.mrope_position_delta = mrope_position_delta
|
||||
|
||||
def _record_kv_event_lora_name(self, req: Req, lora_name: Optional[str]) -> None:
|
||||
# Keyed by the final extra_key, so it must run after elastic namespacing.
|
||||
# Only a cache whose events are taken prunes the table.
|
||||
if (
|
||||
self.kv_events_publisher.enable_kv_cache_events
|
||||
and self.tree_cache.supports_prefix_sharing()
|
||||
):
|
||||
self.kv_event_lora_names.register(req, lora_name)
|
||||
|
||||
def _maybe_namespace_elastic_radix_cache(self, req: Req) -> None:
|
||||
if (
|
||||
get_exec().moe.elastic_ep_backend is None
|
||||
@@ -2990,6 +3001,7 @@ class Scheduler(
|
||||
if recv_req.pp_prefetch_ticketed is True:
|
||||
self.tree_cache.bind_prefetch_ticket(req.rid)
|
||||
self._maybe_namespace_elastic_radix_cache(req)
|
||||
self._record_kv_event_lora_name(req, recv_req.lora_name)
|
||||
|
||||
if mm_input_error is not None:
|
||||
req.set_finish_with_abort(
|
||||
@@ -3566,6 +3578,7 @@ class Scheduler(
|
||||
)
|
||||
req.tokenizer = self.tokenizer
|
||||
self._maybe_namespace_elastic_radix_cache(req)
|
||||
self._record_kv_event_lora_name(req, recv_req.lora_name)
|
||||
|
||||
if mm_input_error is not None:
|
||||
req.set_finish_with_abort(
|
||||
|
||||
@@ -1602,6 +1602,7 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin):
|
||||
bootstrap_port=obj.bootstrap_port,
|
||||
bootstrap_room=bootstrap_room,
|
||||
lora_id=obj.lora_id,
|
||||
lora_name=obj.lora_path,
|
||||
input_embeds=input_embeds,
|
||||
positional_embed_overrides=obj.positional_embed_overrides,
|
||||
session_id=obj.session_id,
|
||||
@@ -1649,6 +1650,7 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin):
|
||||
priority=obj.priority,
|
||||
dimensions=obj.dimensions,
|
||||
lora_id=obj.lora_id,
|
||||
lora_name=obj.lora_path,
|
||||
http_worker_ipc=obj.http_worker_ipc,
|
||||
return_pooled_hidden_states=obj.return_pooled_hidden_states,
|
||||
multi_item_delimiter_indices=obj.multi_item_delimiter_indices,
|
||||
|
||||
@@ -5,9 +5,10 @@ from typing import TYPE_CHECKING, Any, Optional
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.srt.mem_cache.events import KvEventLoraNames
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from sglang.srt.mem_cache.allocator import BaseTokenToKVPoolAllocator
|
||||
from sglang.srt.mem_cache.events import LoRANameTable
|
||||
from sglang.srt.mem_cache.memory_pool import ReqToTokenPool
|
||||
from sglang.srt.mem_cache.unified_cache.components import ComponentType
|
||||
from sglang.srt.mem_cache.unified_cache.components.base import (
|
||||
@@ -35,8 +36,10 @@ class CacheInitParams:
|
||||
enable_metrics: bool = False
|
||||
enable_kv_cache_events: bool = False
|
||||
dynamo_kv_event_format: bool = False
|
||||
# The scheduler holds the same table and adds each adapter when it loads.
|
||||
kv_event_lora_names: Optional[LoRANameTable] = None
|
||||
# LoRA names for KV events; the scheduler registers requests in it.
|
||||
kv_event_lora_names: KvEventLoraNames = dataclasses.field(
|
||||
default_factory=KvEventLoraNames
|
||||
)
|
||||
enable_session_radix_cache: bool = False
|
||||
|
||||
enable_mamba_extra_buffer: bool = False
|
||||
|
||||
@@ -65,12 +65,57 @@ class LoRANameTable:
|
||||
def register(self, *, lora_id: str, lora_name: str) -> None:
|
||||
self._names[lora_id] = lora_name
|
||||
|
||||
def get(self, extra_key: Optional[str]) -> Optional[str]:
|
||||
return self.resolve(extra_key)
|
||||
|
||||
def resolve(self, extra_key: Optional[str]) -> Optional[str]:
|
||||
if not extra_key:
|
||||
return None
|
||||
return self._names.get(extra_key[-_LORA_ID_LEN:])
|
||||
|
||||
|
||||
class KvEventLoraNames(LoRANameTable):
|
||||
"""Public names of selected LoRA namespaces, kept for the publisher lifetime.
|
||||
|
||||
Write-back can publish a host store after the last GPU removal and after
|
||||
its request dies, so request or published-block counts cannot bound this.
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
# Adapter-id registrations keep the existing Dynamo load-time lookup.
|
||||
self._adapter_names: dict[str, str] = {}
|
||||
self._names: dict[str, str] = {}
|
||||
|
||||
def register(
|
||||
self,
|
||||
req: Any = None,
|
||||
lora_name: Optional[str] = None,
|
||||
*,
|
||||
lora_id: Optional[str] = None,
|
||||
) -> None:
|
||||
"""Name ``req``'s namespace; ``req`` needs ``extra_key`` and ``lora_id``."""
|
||||
if lora_id is not None:
|
||||
self._adapter_names[lora_id] = lora_name
|
||||
return
|
||||
if req.lora_id is None:
|
||||
return
|
||||
# Older IPC senders omit the optional name; admission still resolves an ID.
|
||||
lora_name = lora_name or self._adapter_names.get(req.lora_id)
|
||||
if not lora_name:
|
||||
return
|
||||
self._names[req.extra_key] = lora_name
|
||||
|
||||
def get(self, extra_key: Optional[str]) -> Optional[str]:
|
||||
return self._names.get(extra_key)
|
||||
|
||||
def resolve(self, extra_key: Optional[str]) -> Optional[str]:
|
||||
if not extra_key:
|
||||
return None
|
||||
return self._names.get(extra_key) or self._adapter_names.get(
|
||||
extra_key[-_LORA_ID_LEN:]
|
||||
)
|
||||
|
||||
|
||||
class KVCacheEventRecorder:
|
||||
"""Collects KV placement events for one cache.
|
||||
|
||||
@@ -84,12 +129,12 @@ class KVCacheEventRecorder:
|
||||
enabled: bool,
|
||||
page_size: int,
|
||||
dynamo_format: bool = False,
|
||||
lora_names: Optional[LoRANameTable] = None,
|
||||
lora_names: Optional[KvEventLoraNames] = None,
|
||||
):
|
||||
self.enabled = enabled
|
||||
self.page_size = page_size
|
||||
self.dynamo_format = dynamo_format
|
||||
self.lora_names = lora_names if lora_names is not None else LoRANameTable()
|
||||
self.lora_names = lora_names if lora_names is not None else KvEventLoraNames()
|
||||
self._queue: list = []
|
||||
|
||||
def enqueue(self, event) -> None:
|
||||
@@ -114,6 +159,7 @@ class KVCacheEventRecorder:
|
||||
and tail.block_size == event.block_size
|
||||
and tail.cache_salt == event.cache_salt
|
||||
and tail.session_id == event.session_id
|
||||
and tail.lora_name == event.lora_name
|
||||
and tail.block_hashes
|
||||
and event.parent_block_hash == tail.block_hashes[-1]
|
||||
):
|
||||
@@ -151,9 +197,15 @@ class KVCacheEventRecorder:
|
||||
return hash_str_to_int64(parent_hash_values[-1])
|
||||
|
||||
def _namespace_seed(self, node: Any) -> Optional[bytes]:
|
||||
# The default format publishes the hashes unchanged.
|
||||
if not self.dynamo_format:
|
||||
return None
|
||||
return kv_event_namespace_seed(
|
||||
extra_key=(
|
||||
node.key.extra_key
|
||||
if self.lora_names.get(node.key.extra_key) is not None
|
||||
else None
|
||||
),
|
||||
cache_salt=None,
|
||||
)
|
||||
return kv_event_namespace_seed(
|
||||
extra_key=node.key.extra_key, cache_salt=node.key.cache_salt
|
||||
)
|
||||
@@ -177,7 +229,9 @@ class KVCacheEventRecorder:
|
||||
parent_block_hash, namespace_seed=namespace_seed
|
||||
)
|
||||
lora_name = (
|
||||
self.lora_names.resolve(node.key.extra_key) if self.dynamo_format else None
|
||||
self.lora_names.resolve(node.key.extra_key)
|
||||
if self.dynamo_format
|
||||
else self.lora_names.get(node.key.extra_key)
|
||||
)
|
||||
|
||||
page_index = 0
|
||||
|
||||
@@ -40,6 +40,7 @@ from sglang.srt.managers.mm_schedule import init_mm_embedding_cache
|
||||
from sglang.srt.mem_cache.base_swa_memory_pool import BaseSWAKVPool
|
||||
from sglang.srt.mem_cache.cache_init_params import CacheInitParams
|
||||
from sglang.srt.mem_cache.deepseek_v4_memory_pool import DeepSeekV4TokenToKVPool
|
||||
from sglang.srt.mem_cache.events import KvEventLoraNames
|
||||
from sglang.srt.mem_cache.hicache_auto_size import auto_size_hicache
|
||||
from sglang.srt.mem_cache.memory_pool import HybridLinearKVPool, MHATokenToKVPool
|
||||
from sglang.srt.mem_cache.pool_host.base import _WRITE_BACK_STAGING_PAGE_CHUNK
|
||||
@@ -61,7 +62,6 @@ from sglang.srt.utils import ceil_align, is_hip
|
||||
if TYPE_CHECKING:
|
||||
from sglang.srt.configs.model_config import ModelConfig
|
||||
from sglang.srt.managers.tp_worker import BaseTpWorker
|
||||
from sglang.srt.mem_cache.events import LoRANameTable
|
||||
from sglang.srt.server_args import ServerArgs
|
||||
from sglang.srt.speculative.base_spec_worker import HiCacheDraftPlan
|
||||
from sglang.srt.speculative.spec_info import SpeculativeAlgorithm
|
||||
@@ -257,7 +257,7 @@ def build_kv_cache(
|
||||
enable_hierarchical_cache: bool,
|
||||
hicache_draft_plan: Optional[HiCacheDraftPlan] = None,
|
||||
dynamo_kv_event_format: bool = False,
|
||||
kv_event_lora_names: Optional[LoRANameTable] = None,
|
||||
kv_event_lora_names: Optional[KvEventLoraNames] = None,
|
||||
) -> KVCacheBuildResult:
|
||||
parallel = get_parallel()
|
||||
sliding_window_size: Optional[int] = None
|
||||
|
||||
@@ -71,7 +71,7 @@ class RadixKey:
|
||||
):
|
||||
# token ids sequence (raw ints in both modes)
|
||||
self.token_ids = token_ids
|
||||
# Namespaces the tree and storage; omitted from KV events.
|
||||
# Namespaces the tree and storage; KV events see only its LoRA name.
|
||||
self.extra_key = extra_key
|
||||
# Namespaces the tree, storage and KV events.
|
||||
self.cache_salt = cache_salt or None
|
||||
|
||||
@@ -14,6 +14,8 @@ from sglang.srt.disaggregation.kv_events import (
|
||||
BlockRemoved,
|
||||
BlockStored,
|
||||
StorageMedium,
|
||||
kv_event_namespace_seed,
|
||||
namespaced_block_hash,
|
||||
)
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.mem_cache.allocator.swa import is_swa_req_ring
|
||||
@@ -26,7 +28,7 @@ from sglang.srt.mem_cache.base_prefix_cache import (
|
||||
MatchPrefixParams,
|
||||
MatchResult,
|
||||
)
|
||||
from sglang.srt.mem_cache.events import LoRANameTable
|
||||
from sglang.srt.mem_cache.events import KvEventLoraNames
|
||||
from sglang.srt.mem_cache.hicache_storage import PoolHitPolicy, PoolName, PoolTransfer
|
||||
from sglang.srt.mem_cache.radix_cache import RadixKey
|
||||
from sglang.srt.mem_cache.rust_tree_core.extension import bindings
|
||||
@@ -60,7 +62,9 @@ from sglang.srt.mem_cache.unified_cache.unified_tree_core_interface import (
|
||||
TreePathNodeInfo,
|
||||
UnifiedTreeCoreInterface,
|
||||
)
|
||||
from sglang.srt.mem_cache.utils import get_eviction_strategy
|
||||
from sglang.srt.mem_cache.utils import (
|
||||
get_eviction_strategy,
|
||||
)
|
||||
from sglang.srt.runtime_context import get_exec, mamba_cache_chunk_size
|
||||
from sglang.srt.utils import assert_int64_array
|
||||
|
||||
@@ -97,23 +101,54 @@ def _radix_key_buffer(key: RadixKey) -> array:
|
||||
return token_ids
|
||||
|
||||
|
||||
def _kv_event_from_tagged(event: tuple, lora_names: LoRANameTable):
|
||||
"""Build the Python KV cache event for one of the binding's tagged tuples."""
|
||||
def _kv_event_from_tagged(
|
||||
event: tuple, lora_names: KvEventLoraNames, dynamo_format: bool = False
|
||||
):
|
||||
"""Build the Python KV cache event for one of the binding's tagged tuples.
|
||||
|
||||
Rust hashes are LoRA-agnostic; the trailing extra_key selects the LoRA
|
||||
namespace, applied here as in KVCacheEventRecorder.
|
||||
"""
|
||||
tag = event[0]
|
||||
if tag == "block_stored":
|
||||
lora_name = (
|
||||
lora_names.resolve(event[8]) if dynamo_format else lora_names.get(event[8])
|
||||
)
|
||||
seed = (
|
||||
kv_event_namespace_seed(extra_key=event[8], cache_salt=None)
|
||||
if not dynamo_format and lora_name is not None
|
||||
else None
|
||||
)
|
||||
parent = event[2]
|
||||
return BlockStored(
|
||||
block_hashes=event[1],
|
||||
parent_block_hash=event[2],
|
||||
block_hashes=[
|
||||
namespaced_block_hash(h, namespace_seed=seed) for h in event[1]
|
||||
],
|
||||
parent_block_hash=(
|
||||
None
|
||||
if parent is None
|
||||
else namespaced_block_hash(parent, namespace_seed=seed)
|
||||
),
|
||||
token_ids=event[3],
|
||||
block_size=event[4],
|
||||
lora_id=None,
|
||||
medium=StorageMedium(event[5]),
|
||||
cache_salt=event[6],
|
||||
session_id=event[7],
|
||||
lora_name=lora_names.resolve(event[8]),
|
||||
lora_name=lora_name,
|
||||
)
|
||||
if tag == "block_removed":
|
||||
return BlockRemoved(block_hashes=event[1], medium=StorageMedium(event[2]))
|
||||
seed = (
|
||||
kv_event_namespace_seed(extra_key=event[3], cache_salt=None)
|
||||
if not dynamo_format and lora_names.get(event[3]) is not None
|
||||
else None
|
||||
)
|
||||
return BlockRemoved(
|
||||
block_hashes=[
|
||||
namespaced_block_hash(h, namespace_seed=seed) for h in event[1]
|
||||
],
|
||||
medium=StorageMedium(event[2]),
|
||||
)
|
||||
if tag == "all_blocks_cleared":
|
||||
return AllBlocksCleared()
|
||||
raise ValueError(f"unknown kv event tag: {tag}")
|
||||
@@ -305,20 +340,26 @@ class _RustKVCacheEventRecorder:
|
||||
"""Expose the Rust event queue through the Python recorder interface."""
|
||||
|
||||
def __init__(
|
||||
self, binding, enabled: bool, lora_names: Optional[LoRANameTable] = None
|
||||
self,
|
||||
binding,
|
||||
enabled: bool,
|
||||
lora_names: KvEventLoraNames,
|
||||
dynamo_format: bool = False,
|
||||
):
|
||||
self._binding = binding
|
||||
self.enabled = enabled
|
||||
self.lora_names = lora_names if lora_names is not None else LoRANameTable()
|
||||
self._lora_names = lora_names
|
||||
self._dynamo_format = dynamo_format
|
||||
|
||||
def record_all_cleared(self) -> None:
|
||||
self._binding.record_all_cleared_event()
|
||||
|
||||
def take(self) -> list:
|
||||
return [
|
||||
_kv_event_from_tagged(event, self.lora_names)
|
||||
events = [
|
||||
_kv_event_from_tagged(event, self._lora_names, self._dynamo_format)
|
||||
for event in self._binding.take_events()
|
||||
]
|
||||
return events
|
||||
|
||||
|
||||
class RustUnifiedTreeCore(UnifiedTreeCoreInterface):
|
||||
@@ -445,8 +486,9 @@ class RustUnifiedTreeCore(UnifiedTreeCoreInterface):
|
||||
)
|
||||
self.kv_events = _RustKVCacheEventRecorder(
|
||||
self._binding,
|
||||
params.enable_kv_cache_events,
|
||||
enabled=params.enable_kv_cache_events,
|
||||
lora_names=params.kv_event_lora_names,
|
||||
dynamo_format=params.dynamo_kv_event_format,
|
||||
)
|
||||
# The default-root empty result, prebuilt once from the binding.
|
||||
self._empty_match_result = _match_result_from_binding(
|
||||
|
||||
Reference in New Issue
Block a user