fix: preserve QSA indexer state through HiCache (#39893)
This commit is contained in:
@@ -33,6 +33,7 @@ from sglang.srt.mem_cache.pool_host.mha import (
|
||||
get_mha_host_pool_cls,
|
||||
)
|
||||
from sglang.srt.mem_cache.pool_host.mla import MLATokenToKVPoolHost
|
||||
from sglang.srt.mem_cache.pool_host.qsa import QSAIndexerPoolHost
|
||||
from sglang.srt.mem_cache.pool_host.unified import UnifiedPageEnvelopeHostPool
|
||||
from sglang.srt.mem_cache.unified_cache.component_type import ComponentType
|
||||
from sglang.srt.runtime_context import get_memory, get_parallel, get_serving
|
||||
@@ -1128,6 +1129,7 @@ def build_hybrid_mamba_stack(
|
||||
load_cache_event,
|
||||
storage_backend: Optional[str],
|
||||
use_mla: bool,
|
||||
qsa_pool: Any = None,
|
||||
host_mamba_evict_fn: Optional[Callable[[int], Any]] = None,
|
||||
device_mamba_evict_fn: Optional[Callable[[int], Any]] = None,
|
||||
prefetch_threshold: int = 256,
|
||||
@@ -1141,6 +1143,7 @@ def build_hybrid_mamba_stack(
|
||||
mamba_allocator = params.req_to_token_pool.mamba_allocator
|
||||
from sglang.srt.mem_cache.memory_pool import HybridLinearKVPool
|
||||
|
||||
qsa_draft_pools = params.mtp_draft_device_pools if qsa_pool is not None else ()
|
||||
mtp_draft_device_pools = tuple(
|
||||
pool.full_kv_pool if isinstance(pool, HybridLinearKVPool) else pool
|
||||
for pool in params.mtp_draft_device_pools
|
||||
@@ -1150,6 +1153,14 @@ def build_hybrid_mamba_stack(
|
||||
kv_host_size, mamba_host_size = _split_hicache_size(
|
||||
get_memory().hicache_size, (kv_pool, mamba_pool)
|
||||
)
|
||||
if qsa_pool is not None and kv_host_size is not None:
|
||||
kv_bytes = sum(kv_pool.get_kv_size_bytes())
|
||||
index_bytes = sum(
|
||||
b.nbytes
|
||||
for pool in (qsa_pool, *qsa_draft_pools)
|
||||
for b in pool.qsa_compressed_k_buffer_pool
|
||||
)
|
||||
kv_host_size *= kv_bytes / (kv_bytes + index_bytes)
|
||||
kv_host_pool = build_kv_host_pool(
|
||||
kv_pool=kv_pool,
|
||||
page_size=params.page_size,
|
||||
@@ -1202,6 +1213,22 @@ def build_hybrid_mamba_stack(
|
||||
device_free_fn=mamba_allocator.free,
|
||||
),
|
||||
]
|
||||
if qsa_pool is not None:
|
||||
entries.append(
|
||||
build_pool_entry(
|
||||
name=PoolName.INDEXER,
|
||||
host_pool=QSAIndexerPoolHost(
|
||||
qsa_pool,
|
||||
kv_host_pool,
|
||||
allocator_type=_get_allocator_type(),
|
||||
mtp_draft_device_pools=qsa_draft_pools,
|
||||
),
|
||||
device_pool=qsa_pool,
|
||||
layer_mapping=full_layer_mapping,
|
||||
transfer_layer_id_max=transfer_layer_id_max + len(qsa_draft_pools),
|
||||
packed_draft_device_pools=qsa_draft_pools,
|
||||
)
|
||||
)
|
||||
host_pool_group = HostPoolGroup(entries)
|
||||
cache_controller = HybridCacheController(
|
||||
params.token_to_kv_pool_allocator,
|
||||
@@ -1426,6 +1453,7 @@ def build_full_draft_pools(
|
||||
) -> tuple[list[SidecarPoolSpec], list[PoolEntry]]:
|
||||
"""Build draft KV/DSA sidecars whose indices follow target full KV."""
|
||||
from sglang.srt.mem_cache.memory_pool import DSATokenToKVPool, HybridLinearKVPool
|
||||
from sglang.srt.mem_cache.qsa_kv_pool import QSATokenToKVPool
|
||||
|
||||
pool = draft_kv_pool
|
||||
if isinstance(pool, HybridLinearKVPool):
|
||||
@@ -1490,6 +1518,22 @@ def build_full_draft_pools(
|
||||
)
|
||||
)
|
||||
|
||||
if isinstance(draft_kv_pool, QSATokenToKVPool):
|
||||
specs.append(SidecarPoolSpec(PoolName.DRAFT_INDEXER, PoolName.KV))
|
||||
entries.append(
|
||||
build_pool_entry(
|
||||
name=PoolName.DRAFT_INDEXER,
|
||||
host_pool=QSAIndexerPoolHost(
|
||||
draft_kv_pool,
|
||||
draft_host_pool,
|
||||
allocator_type=_get_allocator_type(),
|
||||
),
|
||||
device_pool=draft_kv_pool,
|
||||
layer_mapping=draft_layer_mapping,
|
||||
transfer_layer_id_max=draft_host_pool.layer_num,
|
||||
)
|
||||
)
|
||||
|
||||
return specs, entries
|
||||
|
||||
|
||||
@@ -1770,6 +1814,9 @@ class _MambaStrategy(StackStrategy):
|
||||
model_name=None,
|
||||
enable_storage_metrics=False,
|
||||
):
|
||||
from sglang.srt.mem_cache.qsa_kv_pool import QSATokenToKVPool
|
||||
|
||||
qsa_pool = kvcache if isinstance(kvcache, QSATokenToKVPool) else None
|
||||
full_layer_mapping = _stage_local_layer_mapping(
|
||||
kvcache.full_attention_layer_id_mapping, kvcache.start_layer
|
||||
)
|
||||
@@ -1785,6 +1832,7 @@ class _MambaStrategy(StackStrategy):
|
||||
load_cache_event=load_cache_event,
|
||||
storage_backend=storage_backend,
|
||||
use_mla=kvcache.use_mla,
|
||||
qsa_pool=qsa_pool,
|
||||
host_mamba_evict_fn=lambda n: cache.evict_host(n, ComponentType.MAMBA),
|
||||
device_mamba_evict_fn=lambda n: _evict_mamba_for_device_alloc(cache, n),
|
||||
prefetch_threshold=prefetch_threshold,
|
||||
@@ -1799,8 +1847,13 @@ class _MambaStrategy(StackStrategy):
|
||||
ComponentType.FULL: host_pool_group.get_pool(PoolName.KV),
|
||||
ComponentType.MAMBA: host_pool_group.get_pool(PoolName.MAMBA),
|
||||
},
|
||||
sidecars=(
|
||||
[SidecarPoolSpec(PoolName.INDEXER, PoolName.KV)]
|
||||
if qsa_pool is not None
|
||||
else []
|
||||
),
|
||||
register_req_to_token_counter=True,
|
||||
pools_desc="KV + MAMBA",
|
||||
pools_desc="KV + MAMBA + INDEXER" if qsa_pool is not None else "KV + MAMBA",
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,64 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.srt.mem_cache.memory_pool_host import DeepSeekV4PagedHostPool
|
||||
from sglang.srt.mem_cache.qsa_kv_pool import QSATokenToKVPool
|
||||
|
||||
|
||||
class QSAIndexerPoolHost(DeepSeekV4PagedHostPool):
|
||||
"""Page-row mirror of QSA compressed keys, indexed by full-KV token slots."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
device_pool: QSATokenToKVPool,
|
||||
anchor_host,
|
||||
*,
|
||||
allocator_type: str = "default",
|
||||
mtp_draft_device_pools: tuple[QSATokenToKVPool, ...] = (),
|
||||
):
|
||||
page_size = anchor_host.page_size
|
||||
ratio = device_pool.qsa_compress_ratio
|
||||
if page_size % ratio:
|
||||
raise ValueError(
|
||||
"QSA HiCache pages must be divisible by the compress ratio"
|
||||
)
|
||||
self.device_pool = device_pool
|
||||
pools = (device_pool, *mtp_draft_device_pools)
|
||||
buffers = [b for pool in pools for b in pool.qsa_compressed_k_buffer_pool]
|
||||
if not buffers:
|
||||
raise ValueError("QSA HiCache requires at least one local indexer layer")
|
||||
item_bytes = buffers[0][0].nbytes * (page_size // ratio)
|
||||
if any(
|
||||
pool.qsa_compress_ratio != ratio
|
||||
or any(
|
||||
b[0].nbytes * (page_size // ratio) != item_bytes
|
||||
for b in pool.qsa_compressed_k_buffer_pool
|
||||
)
|
||||
for pool in pools
|
||||
):
|
||||
raise ValueError("Packed QSA indexer pools must share the same page shape")
|
||||
# A full-KV page and its compressed groups share ownership. Packing the
|
||||
# groups into a byte row lets the existing whole-page kernels move both
|
||||
# coordinate spaces using the same full-KV page number.
|
||||
super().__init__(
|
||||
pool_name="qsa_indexer",
|
||||
device_buffers=[
|
||||
b.view(torch.uint8).reshape(-1, item_bytes) for b in buffers
|
||||
],
|
||||
item_bytes=item_bytes,
|
||||
num_host_pages=anchor_host.page_num,
|
||||
slot_page_size=page_size,
|
||||
layout=anchor_host.layout,
|
||||
device=anchor_host.device,
|
||||
pin_memory=anchor_host.pin_memory,
|
||||
allocator_type=allocator_type,
|
||||
page_aligned_only=True,
|
||||
)
|
||||
self.size_per_token = self.get_size_per_token()
|
||||
|
||||
def get_size_per_token(self):
|
||||
return self.layer_num * self.item_bytes // self.slot_page_size
|
||||
|
||||
def get_ksize_per_token(self):
|
||||
return self.get_size_per_token()
|
||||
@@ -252,6 +252,8 @@ class QSATokenToKVPool(HybridLinearKVPool):
|
||||
return self.qsa_rope_position_buffer[loc.long()]
|
||||
|
||||
def get_qsa_compressed_k_buffer(self, layer_id: int) -> torch.Tensor:
|
||||
# The indexer reads compressed keys before attention reads the full KV.
|
||||
self._wait_for_layer(layer_id)
|
||||
return self.qsa_compressed_k_buffer_pool[
|
||||
self._transfer_full_attention_id(layer_id)
|
||||
]
|
||||
|
||||
Reference in New Issue
Block a user