fix: preserve QSA indexer state through HiCache (#39893)

This commit is contained in:
Shuwen Wang
2026-10-02 23:49:38 +08:00
committed by GitHub
parent a74f259f38
commit 89f21671bb
5 changed files with 369 additions and 7 deletions
@@ -33,6 +33,7 @@ from sglang.srt.mem_cache.pool_host.mha import (
get_mha_host_pool_cls,
)
from sglang.srt.mem_cache.pool_host.mla import MLATokenToKVPoolHost
from sglang.srt.mem_cache.pool_host.qsa import QSAIndexerPoolHost
from sglang.srt.mem_cache.pool_host.unified import UnifiedPageEnvelopeHostPool
from sglang.srt.mem_cache.unified_cache.component_type import ComponentType
from sglang.srt.runtime_context import get_memory, get_parallel, get_serving
@@ -1128,6 +1129,7 @@ def build_hybrid_mamba_stack(
load_cache_event,
storage_backend: Optional[str],
use_mla: bool,
qsa_pool: Any = None,
host_mamba_evict_fn: Optional[Callable[[int], Any]] = None,
device_mamba_evict_fn: Optional[Callable[[int], Any]] = None,
prefetch_threshold: int = 256,
@@ -1141,6 +1143,7 @@ def build_hybrid_mamba_stack(
mamba_allocator = params.req_to_token_pool.mamba_allocator
from sglang.srt.mem_cache.memory_pool import HybridLinearKVPool
qsa_draft_pools = params.mtp_draft_device_pools if qsa_pool is not None else ()
mtp_draft_device_pools = tuple(
pool.full_kv_pool if isinstance(pool, HybridLinearKVPool) else pool
for pool in params.mtp_draft_device_pools
@@ -1150,6 +1153,14 @@ def build_hybrid_mamba_stack(
kv_host_size, mamba_host_size = _split_hicache_size(
get_memory().hicache_size, (kv_pool, mamba_pool)
)
if qsa_pool is not None and kv_host_size is not None:
kv_bytes = sum(kv_pool.get_kv_size_bytes())
index_bytes = sum(
b.nbytes
for pool in (qsa_pool, *qsa_draft_pools)
for b in pool.qsa_compressed_k_buffer_pool
)
kv_host_size *= kv_bytes / (kv_bytes + index_bytes)
kv_host_pool = build_kv_host_pool(
kv_pool=kv_pool,
page_size=params.page_size,
@@ -1202,6 +1213,22 @@ def build_hybrid_mamba_stack(
device_free_fn=mamba_allocator.free,
),
]
if qsa_pool is not None:
entries.append(
build_pool_entry(
name=PoolName.INDEXER,
host_pool=QSAIndexerPoolHost(
qsa_pool,
kv_host_pool,
allocator_type=_get_allocator_type(),
mtp_draft_device_pools=qsa_draft_pools,
),
device_pool=qsa_pool,
layer_mapping=full_layer_mapping,
transfer_layer_id_max=transfer_layer_id_max + len(qsa_draft_pools),
packed_draft_device_pools=qsa_draft_pools,
)
)
host_pool_group = HostPoolGroup(entries)
cache_controller = HybridCacheController(
params.token_to_kv_pool_allocator,
@@ -1426,6 +1453,7 @@ def build_full_draft_pools(
) -> tuple[list[SidecarPoolSpec], list[PoolEntry]]:
"""Build draft KV/DSA sidecars whose indices follow target full KV."""
from sglang.srt.mem_cache.memory_pool import DSATokenToKVPool, HybridLinearKVPool
from sglang.srt.mem_cache.qsa_kv_pool import QSATokenToKVPool
pool = draft_kv_pool
if isinstance(pool, HybridLinearKVPool):
@@ -1490,6 +1518,22 @@ def build_full_draft_pools(
)
)
if isinstance(draft_kv_pool, QSATokenToKVPool):
specs.append(SidecarPoolSpec(PoolName.DRAFT_INDEXER, PoolName.KV))
entries.append(
build_pool_entry(
name=PoolName.DRAFT_INDEXER,
host_pool=QSAIndexerPoolHost(
draft_kv_pool,
draft_host_pool,
allocator_type=_get_allocator_type(),
),
device_pool=draft_kv_pool,
layer_mapping=draft_layer_mapping,
transfer_layer_id_max=draft_host_pool.layer_num,
)
)
return specs, entries
@@ -1770,6 +1814,9 @@ class _MambaStrategy(StackStrategy):
model_name=None,
enable_storage_metrics=False,
):
from sglang.srt.mem_cache.qsa_kv_pool import QSATokenToKVPool
qsa_pool = kvcache if isinstance(kvcache, QSATokenToKVPool) else None
full_layer_mapping = _stage_local_layer_mapping(
kvcache.full_attention_layer_id_mapping, kvcache.start_layer
)
@@ -1785,6 +1832,7 @@ class _MambaStrategy(StackStrategy):
load_cache_event=load_cache_event,
storage_backend=storage_backend,
use_mla=kvcache.use_mla,
qsa_pool=qsa_pool,
host_mamba_evict_fn=lambda n: cache.evict_host(n, ComponentType.MAMBA),
device_mamba_evict_fn=lambda n: _evict_mamba_for_device_alloc(cache, n),
prefetch_threshold=prefetch_threshold,
@@ -1799,8 +1847,13 @@ class _MambaStrategy(StackStrategy):
ComponentType.FULL: host_pool_group.get_pool(PoolName.KV),
ComponentType.MAMBA: host_pool_group.get_pool(PoolName.MAMBA),
},
sidecars=(
[SidecarPoolSpec(PoolName.INDEXER, PoolName.KV)]
if qsa_pool is not None
else []
),
register_req_to_token_counter=True,
pools_desc="KV + MAMBA",
pools_desc="KV + MAMBA + INDEXER" if qsa_pool is not None else "KV + MAMBA",
)
@@ -0,0 +1,64 @@
from __future__ import annotations
import torch
from sglang.srt.mem_cache.memory_pool_host import DeepSeekV4PagedHostPool
from sglang.srt.mem_cache.qsa_kv_pool import QSATokenToKVPool
class QSAIndexerPoolHost(DeepSeekV4PagedHostPool):
"""Page-row mirror of QSA compressed keys, indexed by full-KV token slots."""
def __init__(
self,
device_pool: QSATokenToKVPool,
anchor_host,
*,
allocator_type: str = "default",
mtp_draft_device_pools: tuple[QSATokenToKVPool, ...] = (),
):
page_size = anchor_host.page_size
ratio = device_pool.qsa_compress_ratio
if page_size % ratio:
raise ValueError(
"QSA HiCache pages must be divisible by the compress ratio"
)
self.device_pool = device_pool
pools = (device_pool, *mtp_draft_device_pools)
buffers = [b for pool in pools for b in pool.qsa_compressed_k_buffer_pool]
if not buffers:
raise ValueError("QSA HiCache requires at least one local indexer layer")
item_bytes = buffers[0][0].nbytes * (page_size // ratio)
if any(
pool.qsa_compress_ratio != ratio
or any(
b[0].nbytes * (page_size // ratio) != item_bytes
for b in pool.qsa_compressed_k_buffer_pool
)
for pool in pools
):
raise ValueError("Packed QSA indexer pools must share the same page shape")
# A full-KV page and its compressed groups share ownership. Packing the
# groups into a byte row lets the existing whole-page kernels move both
# coordinate spaces using the same full-KV page number.
super().__init__(
pool_name="qsa_indexer",
device_buffers=[
b.view(torch.uint8).reshape(-1, item_bytes) for b in buffers
],
item_bytes=item_bytes,
num_host_pages=anchor_host.page_num,
slot_page_size=page_size,
layout=anchor_host.layout,
device=anchor_host.device,
pin_memory=anchor_host.pin_memory,
allocator_type=allocator_type,
page_aligned_only=True,
)
self.size_per_token = self.get_size_per_token()
def get_size_per_token(self):
return self.layer_num * self.item_bytes // self.slot_page_size
def get_ksize_per_token(self):
return self.get_size_per_token()
@@ -252,6 +252,8 @@ class QSATokenToKVPool(HybridLinearKVPool):
return self.qsa_rope_position_buffer[loc.long()]
def get_qsa_compressed_k_buffer(self, layer_id: int) -> torch.Tensor:
# The indexer reads compressed keys before attention reads the full KV.
self._wait_for_layer(layer_id)
return self.qsa_compressed_k_buffer_pool[
self._transfer_full_attention_id(layer_id)
]