[KV Events] Publish LoRA adapter names and namespaced block hashes (#42455)
Signed-off-by: Banruo Liu <lichangxinlivingshade@gmail.com> Co-authored-by: Shangming Cai <csmthu@gmail.com>
This commit is contained in:
co-authored by
Shangming Cai
parent
97bfa5cc53
commit
d62b9c2480
@@ -19,6 +19,7 @@ KV caching events
|
||||
|
||||
import atexit
|
||||
import enum
|
||||
import hashlib
|
||||
import logging
|
||||
import os
|
||||
import queue
|
||||
@@ -28,7 +29,7 @@ from abc import ABC, abstractmethod
|
||||
from collections import deque
|
||||
from itertools import count
|
||||
from queue import Queue
|
||||
from typing import Any, Callable, Optional, Union
|
||||
from typing import Any, Callable, Literal, Optional, Union
|
||||
|
||||
import msgspec
|
||||
import zmq
|
||||
@@ -289,6 +290,9 @@ class BlockStored(KVCacheEvent):
|
||||
# Session that triggered this store. Attribution only: the blocks may be
|
||||
# shared with other sessions, and the hash does not depend on it.
|
||||
session_id: Optional[str] = None
|
||||
# The LoRA adapter name of the request that stored these blocks.
|
||||
# Set only in the dynamo format.
|
||||
lora_name: Optional[str] = None
|
||||
|
||||
|
||||
class BlockRemoved(KVCacheEvent):
|
||||
@@ -300,6 +304,42 @@ class AllBlocksCleared(KVCacheEvent):
|
||||
pass
|
||||
|
||||
|
||||
# The dynamo format publishes namespaced hashes as block hashes. If you change
|
||||
# this tag, all of these hashes change. unified_tree_core.rs has a Rust copy.
|
||||
_NAMESPACE_SEED_TAG = b"sglang-kv-event-namespace-v1"
|
||||
|
||||
|
||||
def kv_event_namespace_seed(
|
||||
*, extra_key: Optional[str], cache_salt: Optional[str]
|
||||
) -> Optional[bytes]:
|
||||
"""Return the seed for namespaced_block_hash, or None for no namespace."""
|
||||
if extra_key is None and cache_salt is None:
|
||||
return None
|
||||
digest = hashlib.sha256(_NAMESPACE_SEED_TAG)
|
||||
# The presence byte and the length prefix make ("a", "bc") and ("ab", "c") differ.
|
||||
for part in (extra_key, cache_salt):
|
||||
if part is None:
|
||||
digest.update(b"\x00")
|
||||
continue
|
||||
encoded = part.encode("utf-8")
|
||||
digest.update(b"\x01" + len(encoded).to_bytes(8, "little") + encoded)
|
||||
return digest.digest()
|
||||
|
||||
|
||||
def namespaced_block_hash(block_hash: int, *, namespace_seed: Optional[bytes]) -> int:
|
||||
"""Return the namespaced hash of a published block hash.
|
||||
|
||||
If namespace_seed is None, the result is ``block_hash``. If not, the result is
|
||||
the first 8 bytes of SHA-256(seed, block_hash as 8 big-endian bytes), as int64.
|
||||
"""
|
||||
if namespace_seed is None:
|
||||
return block_hash
|
||||
digest = hashlib.sha256(
|
||||
namespace_seed + block_hash.to_bytes(8, "big", signed=True)
|
||||
).digest()
|
||||
return int.from_bytes(digest[:8], "big", signed=True)
|
||||
|
||||
|
||||
class KVEventBatch(EventBatch):
|
||||
events: list[Union[BlockStored, BlockRemoved, AllBlocksCleared]]
|
||||
|
||||
@@ -680,12 +720,24 @@ class KVEventsConfig(BaseModel):
|
||||
this topic to receive events.
|
||||
"""
|
||||
|
||||
format: Literal["default", "dynamo"] = "default"
|
||||
"""The event format. "default" is the SGLang format. "dynamo" is the format
|
||||
the Dynamo KV router reads: block hashes also depend on extra_key and
|
||||
cache_salt (see ``namespaced_block_hash``), and BlockStored has lora_name.
|
||||
"""
|
||||
|
||||
@classmethod
|
||||
def from_cli(cls, cli_value: str) -> "KVEventsConfig":
|
||||
"""Parse the CLI value for the event publisher config."""
|
||||
return KVEventsConfig.model_validate_json(cli_value)
|
||||
|
||||
|
||||
def uses_dynamo_format(kv_events_config: Optional[str]) -> bool:
|
||||
if not kv_events_config:
|
||||
return False
|
||||
return KVEventsConfig.from_cli(kv_events_config).format == "dynamo"
|
||||
|
||||
|
||||
class EventPublisherFactory:
|
||||
_registry: dict[str, Callable[..., EventPublisher]] = {
|
||||
"null": NullEventPublisher,
|
||||
@@ -705,6 +757,8 @@ class EventPublisherFactory:
|
||||
return NullEventPublisher()
|
||||
config = KVEventsConfig.from_cli(config)
|
||||
config_dict = config.model_dump()
|
||||
# The cache event recorder uses the format. The publisher does not accept it.
|
||||
config_dict.pop("format")
|
||||
|
||||
kind = config_dict.pop("publisher", "null")
|
||||
try:
|
||||
|
||||
@@ -87,6 +87,7 @@ from sglang.srt.disaggregation.decode_kvcache_offload_manager import (
|
||||
DecodeKVCacheOffloadManager,
|
||||
)
|
||||
from sglang.srt.disaggregation.encoder.receiver import create_mm_receiver
|
||||
from sglang.srt.disaggregation.kv_events import uses_dynamo_format
|
||||
from sglang.srt.disaggregation.prefill import (
|
||||
PrefillBootstrapQueue,
|
||||
SchedulerDisaggregationPrefillMixin,
|
||||
@@ -297,6 +298,7 @@ from sglang.srt.mem_cache.common import (
|
||||
discard_kv_cache_backup,
|
||||
release_kv_cache,
|
||||
)
|
||||
from sglang.srt.mem_cache.events import LoRANameTable
|
||||
from sglang.srt.model_executor.forward_batch_info import PPProxyTensors
|
||||
from sglang.srt.model_executor.runner_utils.pool import prewarm_graph_pool_borrow
|
||||
from sglang.srt.model_loader.utils import get_resolved_model_impl
|
||||
@@ -564,6 +566,7 @@ class Scheduler(
|
||||
time.sleep(t)
|
||||
|
||||
# Init cache and memory pool
|
||||
self.init_kv_event_lora_names()
|
||||
result = kv_cache_builder.build_kv_cache(
|
||||
server_args=self.server_args,
|
||||
model_config=self.model_config,
|
||||
@@ -577,6 +580,10 @@ class Scheduler(
|
||||
and get_parallel().attn_tp_rank == 0
|
||||
and get_parallel().attn_cp_rank == 0
|
||||
),
|
||||
dynamo_kv_event_format=uses_dynamo_format(
|
||||
get_observability().kv_events_config
|
||||
),
|
||||
kv_event_lora_names=self.kv_event_lora_names,
|
||||
enable_hierarchical_cache=self.enable_hierarchical_cache,
|
||||
hicache_draft_plan=(
|
||||
self.draft_worker.hicache_draft_plan
|
||||
@@ -2429,6 +2436,10 @@ class Scheduler(
|
||||
groups += [self.pp_group]
|
||||
rank_consensus_checker.configure(groups)
|
||||
|
||||
def init_kv_event_lora_names(self) -> None:
|
||||
"""Init the table that maps lora_id to adapter name for KV events."""
|
||||
self.kv_event_lora_names = LoRANameTable.from_lora_refs(get_lora().lora_paths)
|
||||
|
||||
def init_kv_events_publisher(self) -> None:
|
||||
self.kv_events_publisher = SchedulerKvEventsPublisher(
|
||||
kv_events_config=get_observability().kv_events_config,
|
||||
@@ -5768,6 +5779,10 @@ class Scheduler(
|
||||
"""In-place loading a new lora adapter from disk or huggingface."""
|
||||
|
||||
result = self.tp_worker.load_lora_adapter(recv_req)
|
||||
if result.success:
|
||||
self.kv_event_lora_names.register(
|
||||
lora_id=recv_req.lora_id, lora_name=recv_req.lora_name
|
||||
)
|
||||
return result
|
||||
|
||||
def load_lora_adapter_from_tensors(
|
||||
@@ -5776,6 +5791,10 @@ class Scheduler(
|
||||
"""In-place loading a new lora adapter from serialized tensors."""
|
||||
|
||||
result = self.tp_worker.load_lora_adapter_from_tensors(recv_req)
|
||||
if result.success:
|
||||
self.kv_event_lora_names.register(
|
||||
lora_id=recv_req.lora_id, lora_name=recv_req.lora_name
|
||||
)
|
||||
return result
|
||||
|
||||
def unload_lora_adapter(
|
||||
|
||||
@@ -7,6 +7,7 @@ import torch
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from sglang.srt.mem_cache.allocator import BaseTokenToKVPoolAllocator
|
||||
from sglang.srt.mem_cache.events import LoRANameTable
|
||||
from sglang.srt.mem_cache.memory_pool import ReqToTokenPool
|
||||
from sglang.srt.mem_cache.unified_cache.components import ComponentType
|
||||
from sglang.srt.mem_cache.unified_cache.components.base import (
|
||||
@@ -33,6 +34,9 @@ class CacheInitParams:
|
||||
|
||||
enable_metrics: bool = False
|
||||
enable_kv_cache_events: bool = False
|
||||
dynamo_kv_event_format: bool = False
|
||||
# The scheduler holds the same table and adds each adapter when it loads.
|
||||
kv_event_lora_names: Optional[LoRANameTable] = None
|
||||
enable_session_radix_cache: bool = False
|
||||
|
||||
enable_mamba_extra_buffer: bool = False
|
||||
|
||||
@@ -18,13 +18,17 @@ consumed by KV-aware routers (e.g. dynamo). A cache holds one recorder and calls
|
||||
it; the recorder owns the queue and needs nothing back from its owner.
|
||||
"""
|
||||
|
||||
from typing import Any, Optional
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING, Any, Iterable, Optional
|
||||
|
||||
from sglang.srt.disaggregation.kv_events import (
|
||||
AllBlocksCleared,
|
||||
BlockRemoved,
|
||||
BlockStored,
|
||||
StorageMedium,
|
||||
kv_event_namespace_seed,
|
||||
namespaced_block_hash,
|
||||
)
|
||||
from sglang.srt.mem_cache.utils import (
|
||||
compute_node_event_hash_values,
|
||||
@@ -32,6 +36,40 @@ from sglang.srt.mem_cache.utils import (
|
||||
hash_str_to_int64,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from sglang.srt.lora.lora_registry import LoRARef
|
||||
|
||||
# Length of LoRARef.lora_id, a uuid4 or uuid5 hex string. Req adds the id to the
|
||||
# end of extra_key, so the last 32 characters of extra_key are the id.
|
||||
_LORA_ID_LEN = 32
|
||||
|
||||
|
||||
class LoRANameTable:
|
||||
"""Find the LoRA adapter name from the lora_id at the end of extra_key.
|
||||
|
||||
Req adds lora_id to the end of extra_key, so a tree node has only the id.
|
||||
The table keeps all entries. A lora_id is not used again, and nodes of an
|
||||
unloaded adapter can publish stores until eviction.
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
self._names: dict[str, str] = {}
|
||||
|
||||
@classmethod
|
||||
def from_lora_refs(cls, lora_refs: Optional[Iterable[LoRARef]]) -> LoRANameTable:
|
||||
table = cls()
|
||||
for lora_ref in lora_refs or ():
|
||||
table.register(lora_id=lora_ref.lora_id, lora_name=lora_ref.lora_name)
|
||||
return table
|
||||
|
||||
def register(self, *, lora_id: str, lora_name: str) -> None:
|
||||
self._names[lora_id] = lora_name
|
||||
|
||||
def resolve(self, extra_key: Optional[str]) -> Optional[str]:
|
||||
if not extra_key:
|
||||
return None
|
||||
return self._names.get(extra_key[-_LORA_ID_LEN:])
|
||||
|
||||
|
||||
class KVCacheEventRecorder:
|
||||
"""Collects KV placement events for one cache.
|
||||
@@ -40,9 +78,18 @@ class KVCacheEventRecorder:
|
||||
empty list, so callers never have to guard.
|
||||
"""
|
||||
|
||||
def __init__(self, *, enabled: bool, page_size: int):
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
enabled: bool,
|
||||
page_size: int,
|
||||
dynamo_format: bool = False,
|
||||
lora_names: Optional[LoRANameTable] = None,
|
||||
):
|
||||
self.enabled = enabled
|
||||
self.page_size = page_size
|
||||
self.dynamo_format = dynamo_format
|
||||
self.lora_names = lora_names if lora_names is not None else LoRANameTable()
|
||||
self._queue: list = []
|
||||
|
||||
def enqueue(self, event) -> None:
|
||||
@@ -103,6 +150,14 @@ class KVCacheEventRecorder:
|
||||
return None
|
||||
return hash_str_to_int64(parent_hash_values[-1])
|
||||
|
||||
def _namespace_seed(self, node: Any) -> Optional[bytes]:
|
||||
# The default format publishes the hashes unchanged.
|
||||
if not self.dynamo_format:
|
||||
return None
|
||||
return kv_event_namespace_seed(
|
||||
extra_key=node.key.extra_key, cache_salt=node.key.cache_salt
|
||||
)
|
||||
|
||||
def record_store(
|
||||
self, node: Any, medium=None, *, session_id: Optional[str] = None
|
||||
) -> None:
|
||||
@@ -115,7 +170,15 @@ class KVCacheEventRecorder:
|
||||
medium = StorageMedium.GPU
|
||||
|
||||
event_hash_values = self._node_event_hash_values(node)
|
||||
namespace_seed = self._namespace_seed(node)
|
||||
parent_block_hash = self._parent_block_hash(node)
|
||||
if parent_block_hash is not None:
|
||||
parent_block_hash = namespaced_block_hash(
|
||||
parent_block_hash, namespace_seed=namespace_seed
|
||||
)
|
||||
lora_name = (
|
||||
self.lora_names.resolve(node.key.extra_key) if self.dynamo_format else None
|
||||
)
|
||||
|
||||
page_index = 0
|
||||
logical_len = len(node.key)
|
||||
@@ -131,7 +194,10 @@ class KVCacheEventRecorder:
|
||||
else:
|
||||
page_tokens = list(raw[start:end])
|
||||
|
||||
block_hash = hash_str_to_int64(event_hash_values[page_index])
|
||||
block_hash = namespaced_block_hash(
|
||||
hash_str_to_int64(event_hash_values[page_index]),
|
||||
namespace_seed=namespace_seed,
|
||||
)
|
||||
|
||||
self.enqueue(
|
||||
BlockStored(
|
||||
@@ -143,6 +209,7 @@ class KVCacheEventRecorder:
|
||||
medium=medium,
|
||||
cache_salt=node.key.cache_salt,
|
||||
session_id=session_id,
|
||||
lora_name=lora_name,
|
||||
)
|
||||
)
|
||||
|
||||
@@ -160,6 +227,7 @@ class KVCacheEventRecorder:
|
||||
|
||||
# Hash values must match what was stored.
|
||||
event_hash_values = self._node_event_hash_values(node)
|
||||
namespace_seed = self._namespace_seed(node)
|
||||
|
||||
block_hashes = []
|
||||
logical_len = len(node.key)
|
||||
@@ -169,7 +237,12 @@ class KVCacheEventRecorder:
|
||||
if end <= start:
|
||||
continue
|
||||
|
||||
block_hashes.append(hash_str_to_int64(event_hash_values[page_index]))
|
||||
block_hashes.append(
|
||||
namespaced_block_hash(
|
||||
hash_str_to_int64(event_hash_values[page_index]),
|
||||
namespace_seed=namespace_seed,
|
||||
)
|
||||
)
|
||||
page_index += 1
|
||||
|
||||
if block_hashes:
|
||||
|
||||
@@ -61,6 +61,7 @@ from sglang.srt.utils import ceil_align, is_hip
|
||||
if TYPE_CHECKING:
|
||||
from sglang.srt.configs.model_config import ModelConfig
|
||||
from sglang.srt.managers.tp_worker import BaseTpWorker
|
||||
from sglang.srt.mem_cache.events import LoRANameTable
|
||||
from sglang.srt.server_args import ServerArgs
|
||||
from sglang.srt.speculative.base_spec_worker import HiCacheDraftPlan
|
||||
from sglang.srt.speculative.spec_info import SpeculativeAlgorithm
|
||||
@@ -255,6 +256,8 @@ def build_kv_cache(
|
||||
enable_kv_cache_events: bool,
|
||||
enable_hierarchical_cache: bool,
|
||||
hicache_draft_plan: Optional[HiCacheDraftPlan] = None,
|
||||
dynamo_kv_event_format: bool = False,
|
||||
kv_event_lora_names: Optional[LoRANameTable] = None,
|
||||
) -> KVCacheBuildResult:
|
||||
parallel = get_parallel()
|
||||
sliding_window_size: Optional[int] = None
|
||||
@@ -346,6 +349,8 @@ def build_kv_cache(
|
||||
eviction_policy_config=get_memory().radix_eviction_policy_config,
|
||||
enable_metrics=enable_metrics,
|
||||
enable_kv_cache_events=enable_kv_cache_events,
|
||||
dynamo_kv_event_format=dynamo_kv_event_format,
|
||||
kv_event_lora_names=kv_event_lora_names,
|
||||
enable_session_radix_cache=get_memory().enable_session_radix_cache,
|
||||
enable_mamba_extra_buffer=get_exec().mamba.enable_mamba_extra_buffer,
|
||||
enable_mamba_extra_buffer_lazy=get_exec().mamba.enable_mamba_extra_buffer_lazy,
|
||||
|
||||
@@ -45,7 +45,7 @@ from sglang.srt.mem_cache.base_prefix_cache import (
|
||||
MatchPrefixParams,
|
||||
MatchResult,
|
||||
)
|
||||
from sglang.srt.mem_cache.events import KVCacheEventRecorder
|
||||
from sglang.srt.mem_cache.events import KVCacheEventRecorder, LoRANameTable
|
||||
from sglang.srt.mem_cache.utils import (
|
||||
get_eviction_strategy,
|
||||
split_node_hash_value,
|
||||
@@ -292,6 +292,8 @@ class RadixCache(BasePrefixCache):
|
||||
self.kv_events = KVCacheEventRecorder(
|
||||
enabled=params.enable_kv_cache_events and not self.disable,
|
||||
page_size=self.page_size,
|
||||
dynamo_format=params.dynamo_kv_event_format,
|
||||
lora_names=params.kv_event_lora_names,
|
||||
)
|
||||
|
||||
if params.enable_metrics:
|
||||
@@ -321,6 +323,8 @@ class RadixCache(BasePrefixCache):
|
||||
mock_allocator: Optional[Any] = None,
|
||||
page_size: int = 1,
|
||||
enable_kv_cache_events: bool = False,
|
||||
dynamo_kv_event_format: bool = False,
|
||||
kv_event_lora_names: Optional[LoRANameTable] = None,
|
||||
) -> RadixCache:
|
||||
"""Init a radix cache without memory pools for simulation purpose."""
|
||||
params = CacheInitParams(
|
||||
@@ -329,6 +333,8 @@ class RadixCache(BasePrefixCache):
|
||||
token_to_kv_pool_allocator=mock_allocator,
|
||||
page_size=page_size,
|
||||
enable_kv_cache_events=enable_kv_cache_events,
|
||||
dynamo_kv_event_format=dynamo_kv_event_format,
|
||||
kv_event_lora_names=kv_event_lora_names,
|
||||
)
|
||||
return RadixCache(params)
|
||||
|
||||
|
||||
@@ -26,6 +26,7 @@ from sglang.srt.mem_cache.base_prefix_cache import (
|
||||
MatchPrefixParams,
|
||||
MatchResult,
|
||||
)
|
||||
from sglang.srt.mem_cache.events import LoRANameTable
|
||||
from sglang.srt.mem_cache.hicache_storage import PoolHitPolicy, PoolName, PoolTransfer
|
||||
from sglang.srt.mem_cache.radix_cache import RadixKey
|
||||
from sglang.srt.mem_cache.rust_tree_core.extension import bindings
|
||||
@@ -95,7 +96,7 @@ def _radix_key_buffer(key: RadixKey) -> array:
|
||||
return token_ids
|
||||
|
||||
|
||||
def _kv_event_from_tagged(event: tuple):
|
||||
def _kv_event_from_tagged(event: tuple, lora_names: LoRANameTable):
|
||||
"""Build the Python KV cache event for one of the binding's tagged tuples."""
|
||||
tag = event[0]
|
||||
if tag == "block_stored":
|
||||
@@ -108,6 +109,7 @@ def _kv_event_from_tagged(event: tuple):
|
||||
medium=StorageMedium(event[5]),
|
||||
cache_salt=event[6],
|
||||
session_id=event[7],
|
||||
lora_name=lora_names.resolve(event[8]),
|
||||
)
|
||||
if tag == "block_removed":
|
||||
return BlockRemoved(block_hashes=event[1], medium=StorageMedium(event[2]))
|
||||
@@ -301,15 +303,21 @@ def _fill_evict_result(binding_result, result):
|
||||
class _RustKVCacheEventRecorder:
|
||||
"""Expose the Rust event queue through the Python recorder interface."""
|
||||
|
||||
def __init__(self, binding, enabled: bool):
|
||||
def __init__(
|
||||
self, binding, enabled: bool, lora_names: Optional[LoRANameTable] = None
|
||||
):
|
||||
self._binding = binding
|
||||
self.enabled = enabled
|
||||
self.lora_names = lora_names if lora_names is not None else LoRANameTable()
|
||||
|
||||
def record_all_cleared(self) -> None:
|
||||
self._binding.record_all_cleared_event()
|
||||
|
||||
def take(self) -> list:
|
||||
return [_kv_event_from_tagged(event) for event in self._binding.take_events()]
|
||||
return [
|
||||
_kv_event_from_tagged(event, self.lora_names)
|
||||
for event in self._binding.take_events()
|
||||
]
|
||||
|
||||
|
||||
class RustUnifiedTreeCore(UnifiedTreeCoreInterface):
|
||||
@@ -422,6 +430,7 @@ class RustUnifiedTreeCore(UnifiedTreeCoreInterface):
|
||||
swa_sliding_window_size=params.sliding_window_size,
|
||||
swa_req_ring=is_swa_req_ring(self._allocator),
|
||||
enable_kv_cache_events=params.enable_kv_cache_events,
|
||||
dynamo_kv_event_format=params.dynamo_kv_event_format,
|
||||
mamba_cache_chunk_size=(
|
||||
mamba_cache_chunk_size() if has_mamba else None
|
||||
),
|
||||
@@ -434,7 +443,9 @@ class RustUnifiedTreeCore(UnifiedTreeCoreInterface):
|
||||
[int(component) for component in self.tree_components],
|
||||
)
|
||||
self.kv_events = _RustKVCacheEventRecorder(
|
||||
self._binding, params.enable_kv_cache_events
|
||||
self._binding,
|
||||
params.enable_kv_cache_events,
|
||||
lora_names=params.kv_event_lora_names,
|
||||
)
|
||||
# The default-root empty result, prebuilt once from the binding.
|
||||
self._empty_match_result = _match_result_from_binding(
|
||||
|
||||
@@ -460,7 +460,10 @@ class UnifiedTreeCore(UnifiedTreeCoreInterface):
|
||||
)
|
||||
|
||||
self.kv_events = KVCacheEventRecorder(
|
||||
enabled=params.enable_kv_cache_events, page_size=self.page_size
|
||||
enabled=params.enable_kv_cache_events,
|
||||
page_size=self.page_size,
|
||||
dynamo_format=params.dynamo_kv_event_format,
|
||||
lora_names=params.kv_event_lora_names,
|
||||
)
|
||||
|
||||
self.reset()
|
||||
|
||||
Reference in New Issue
Block a user