fix: preserve QSA indexer state through HiCache (#39893)
This commit is contained in:
@@ -33,6 +33,7 @@ from sglang.srt.mem_cache.pool_host.mha import (
|
||||
get_mha_host_pool_cls,
|
||||
)
|
||||
from sglang.srt.mem_cache.pool_host.mla import MLATokenToKVPoolHost
|
||||
from sglang.srt.mem_cache.pool_host.qsa import QSAIndexerPoolHost
|
||||
from sglang.srt.mem_cache.pool_host.unified import UnifiedPageEnvelopeHostPool
|
||||
from sglang.srt.mem_cache.unified_cache.component_type import ComponentType
|
||||
from sglang.srt.runtime_context import get_memory, get_parallel, get_serving
|
||||
@@ -1128,6 +1129,7 @@ def build_hybrid_mamba_stack(
|
||||
load_cache_event,
|
||||
storage_backend: Optional[str],
|
||||
use_mla: bool,
|
||||
qsa_pool: Any = None,
|
||||
host_mamba_evict_fn: Optional[Callable[[int], Any]] = None,
|
||||
device_mamba_evict_fn: Optional[Callable[[int], Any]] = None,
|
||||
prefetch_threshold: int = 256,
|
||||
@@ -1141,6 +1143,7 @@ def build_hybrid_mamba_stack(
|
||||
mamba_allocator = params.req_to_token_pool.mamba_allocator
|
||||
from sglang.srt.mem_cache.memory_pool import HybridLinearKVPool
|
||||
|
||||
qsa_draft_pools = params.mtp_draft_device_pools if qsa_pool is not None else ()
|
||||
mtp_draft_device_pools = tuple(
|
||||
pool.full_kv_pool if isinstance(pool, HybridLinearKVPool) else pool
|
||||
for pool in params.mtp_draft_device_pools
|
||||
@@ -1150,6 +1153,14 @@ def build_hybrid_mamba_stack(
|
||||
kv_host_size, mamba_host_size = _split_hicache_size(
|
||||
get_memory().hicache_size, (kv_pool, mamba_pool)
|
||||
)
|
||||
if qsa_pool is not None and kv_host_size is not None:
|
||||
kv_bytes = sum(kv_pool.get_kv_size_bytes())
|
||||
index_bytes = sum(
|
||||
b.nbytes
|
||||
for pool in (qsa_pool, *qsa_draft_pools)
|
||||
for b in pool.qsa_compressed_k_buffer_pool
|
||||
)
|
||||
kv_host_size *= kv_bytes / (kv_bytes + index_bytes)
|
||||
kv_host_pool = build_kv_host_pool(
|
||||
kv_pool=kv_pool,
|
||||
page_size=params.page_size,
|
||||
@@ -1202,6 +1213,22 @@ def build_hybrid_mamba_stack(
|
||||
device_free_fn=mamba_allocator.free,
|
||||
),
|
||||
]
|
||||
if qsa_pool is not None:
|
||||
entries.append(
|
||||
build_pool_entry(
|
||||
name=PoolName.INDEXER,
|
||||
host_pool=QSAIndexerPoolHost(
|
||||
qsa_pool,
|
||||
kv_host_pool,
|
||||
allocator_type=_get_allocator_type(),
|
||||
mtp_draft_device_pools=qsa_draft_pools,
|
||||
),
|
||||
device_pool=qsa_pool,
|
||||
layer_mapping=full_layer_mapping,
|
||||
transfer_layer_id_max=transfer_layer_id_max + len(qsa_draft_pools),
|
||||
packed_draft_device_pools=qsa_draft_pools,
|
||||
)
|
||||
)
|
||||
host_pool_group = HostPoolGroup(entries)
|
||||
cache_controller = HybridCacheController(
|
||||
params.token_to_kv_pool_allocator,
|
||||
@@ -1426,6 +1453,7 @@ def build_full_draft_pools(
|
||||
) -> tuple[list[SidecarPoolSpec], list[PoolEntry]]:
|
||||
"""Build draft KV/DSA sidecars whose indices follow target full KV."""
|
||||
from sglang.srt.mem_cache.memory_pool import DSATokenToKVPool, HybridLinearKVPool
|
||||
from sglang.srt.mem_cache.qsa_kv_pool import QSATokenToKVPool
|
||||
|
||||
pool = draft_kv_pool
|
||||
if isinstance(pool, HybridLinearKVPool):
|
||||
@@ -1490,6 +1518,22 @@ def build_full_draft_pools(
|
||||
)
|
||||
)
|
||||
|
||||
if isinstance(draft_kv_pool, QSATokenToKVPool):
|
||||
specs.append(SidecarPoolSpec(PoolName.DRAFT_INDEXER, PoolName.KV))
|
||||
entries.append(
|
||||
build_pool_entry(
|
||||
name=PoolName.DRAFT_INDEXER,
|
||||
host_pool=QSAIndexerPoolHost(
|
||||
draft_kv_pool,
|
||||
draft_host_pool,
|
||||
allocator_type=_get_allocator_type(),
|
||||
),
|
||||
device_pool=draft_kv_pool,
|
||||
layer_mapping=draft_layer_mapping,
|
||||
transfer_layer_id_max=draft_host_pool.layer_num,
|
||||
)
|
||||
)
|
||||
|
||||
return specs, entries
|
||||
|
||||
|
||||
@@ -1770,6 +1814,9 @@ class _MambaStrategy(StackStrategy):
|
||||
model_name=None,
|
||||
enable_storage_metrics=False,
|
||||
):
|
||||
from sglang.srt.mem_cache.qsa_kv_pool import QSATokenToKVPool
|
||||
|
||||
qsa_pool = kvcache if isinstance(kvcache, QSATokenToKVPool) else None
|
||||
full_layer_mapping = _stage_local_layer_mapping(
|
||||
kvcache.full_attention_layer_id_mapping, kvcache.start_layer
|
||||
)
|
||||
@@ -1785,6 +1832,7 @@ class _MambaStrategy(StackStrategy):
|
||||
load_cache_event=load_cache_event,
|
||||
storage_backend=storage_backend,
|
||||
use_mla=kvcache.use_mla,
|
||||
qsa_pool=qsa_pool,
|
||||
host_mamba_evict_fn=lambda n: cache.evict_host(n, ComponentType.MAMBA),
|
||||
device_mamba_evict_fn=lambda n: _evict_mamba_for_device_alloc(cache, n),
|
||||
prefetch_threshold=prefetch_threshold,
|
||||
@@ -1799,8 +1847,13 @@ class _MambaStrategy(StackStrategy):
|
||||
ComponentType.FULL: host_pool_group.get_pool(PoolName.KV),
|
||||
ComponentType.MAMBA: host_pool_group.get_pool(PoolName.MAMBA),
|
||||
},
|
||||
sidecars=(
|
||||
[SidecarPoolSpec(PoolName.INDEXER, PoolName.KV)]
|
||||
if qsa_pool is not None
|
||||
else []
|
||||
),
|
||||
register_req_to_token_counter=True,
|
||||
pools_desc="KV + MAMBA",
|
||||
pools_desc="KV + MAMBA + INDEXER" if qsa_pool is not None else "KV + MAMBA",
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,64 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.srt.mem_cache.memory_pool_host import DeepSeekV4PagedHostPool
|
||||
from sglang.srt.mem_cache.qsa_kv_pool import QSATokenToKVPool
|
||||
|
||||
|
||||
class QSAIndexerPoolHost(DeepSeekV4PagedHostPool):
|
||||
"""Page-row mirror of QSA compressed keys, indexed by full-KV token slots."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
device_pool: QSATokenToKVPool,
|
||||
anchor_host,
|
||||
*,
|
||||
allocator_type: str = "default",
|
||||
mtp_draft_device_pools: tuple[QSATokenToKVPool, ...] = (),
|
||||
):
|
||||
page_size = anchor_host.page_size
|
||||
ratio = device_pool.qsa_compress_ratio
|
||||
if page_size % ratio:
|
||||
raise ValueError(
|
||||
"QSA HiCache pages must be divisible by the compress ratio"
|
||||
)
|
||||
self.device_pool = device_pool
|
||||
pools = (device_pool, *mtp_draft_device_pools)
|
||||
buffers = [b for pool in pools for b in pool.qsa_compressed_k_buffer_pool]
|
||||
if not buffers:
|
||||
raise ValueError("QSA HiCache requires at least one local indexer layer")
|
||||
item_bytes = buffers[0][0].nbytes * (page_size // ratio)
|
||||
if any(
|
||||
pool.qsa_compress_ratio != ratio
|
||||
or any(
|
||||
b[0].nbytes * (page_size // ratio) != item_bytes
|
||||
for b in pool.qsa_compressed_k_buffer_pool
|
||||
)
|
||||
for pool in pools
|
||||
):
|
||||
raise ValueError("Packed QSA indexer pools must share the same page shape")
|
||||
# A full-KV page and its compressed groups share ownership. Packing the
|
||||
# groups into a byte row lets the existing whole-page kernels move both
|
||||
# coordinate spaces using the same full-KV page number.
|
||||
super().__init__(
|
||||
pool_name="qsa_indexer",
|
||||
device_buffers=[
|
||||
b.view(torch.uint8).reshape(-1, item_bytes) for b in buffers
|
||||
],
|
||||
item_bytes=item_bytes,
|
||||
num_host_pages=anchor_host.page_num,
|
||||
slot_page_size=page_size,
|
||||
layout=anchor_host.layout,
|
||||
device=anchor_host.device,
|
||||
pin_memory=anchor_host.pin_memory,
|
||||
allocator_type=allocator_type,
|
||||
page_aligned_only=True,
|
||||
)
|
||||
self.size_per_token = self.get_size_per_token()
|
||||
|
||||
def get_size_per_token(self):
|
||||
return self.layer_num * self.item_bytes // self.slot_page_size
|
||||
|
||||
def get_ksize_per_token(self):
|
||||
return self.get_size_per_token()
|
||||
@@ -252,6 +252,8 @@ class QSATokenToKVPool(HybridLinearKVPool):
|
||||
return self.qsa_rope_position_buffer[loc.long()]
|
||||
|
||||
def get_qsa_compressed_k_buffer(self, layer_id: int) -> torch.Tensor:
|
||||
# The indexer reads compressed keys before attention reads the full KV.
|
||||
self._wait_for_layer(layer_id)
|
||||
return self.qsa_compressed_k_buffer_pool[
|
||||
self._transfer_full_attention_id(layer_id)
|
||||
]
|
||||
|
||||
@@ -0,0 +1,239 @@
|
||||
"""Exercise real QSA host transfers and File L3 with reordered physical pages."""
|
||||
|
||||
import tempfile
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from sglang.srt.managers.cache_controller import LayerDoneCounter
|
||||
from sglang.srt.mem_cache.hicache_storage import (
|
||||
HiCacheFile,
|
||||
HiCacheStorageConfig,
|
||||
PoolName,
|
||||
PoolTransfer,
|
||||
)
|
||||
from sglang.srt.mem_cache.l2_transfer import L2Transfer, L2TransferEngine
|
||||
from sglang.srt.mem_cache.pool_host.mha import MHATokenToKVPoolHost
|
||||
from sglang.srt.mem_cache.pool_host.qsa import QSAIndexerPoolHost
|
||||
from sglang.srt.mem_cache.qsa_kv_pool import QSATokenToKVPool
|
||||
from sglang.srt.runtime_context import publish, reset_context
|
||||
from sglang.srt.server_args import ServerArgs
|
||||
from sglang.test.ci.ci_register import register_cuda_ci
|
||||
|
||||
register_cuda_ci(est_time=30, stage="base-b-kernel-unit", runner_config="4-gpu-b200")
|
||||
|
||||
PAGE = 64
|
||||
RATIO = 4
|
||||
|
||||
|
||||
def make_device_pool(layers, start_layer=0):
|
||||
return QSATokenToKVPool(
|
||||
size=PAGE * 8,
|
||||
dtype=torch.bfloat16,
|
||||
page_size=PAGE,
|
||||
head_num=1,
|
||||
head_dim=128,
|
||||
full_attention_layer_ids=layers,
|
||||
start_layer=start_layer,
|
||||
device="cuda",
|
||||
mamba_pool=None,
|
||||
qsa_index_kv_heads=1,
|
||||
qsa_index_head_dim=128,
|
||||
qsa_compress_ratio=RATIO,
|
||||
qsa_token_topk=128,
|
||||
num_request_slots=4,
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def pools(request):
|
||||
layout, backend = request.param
|
||||
publish(ServerArgs(model_path="dummy"), role="test")
|
||||
device = make_device_pool([7, 11], start_layer=4)
|
||||
anchor = MHATokenToKVPoolHost(device.full_kv_pool, 2, 0, PAGE, layout)
|
||||
host = QSAIndexerPoolHost(device, anchor)
|
||||
try:
|
||||
yield device, anchor, host, backend
|
||||
finally:
|
||||
torch.cuda.synchronize()
|
||||
host.destroy()
|
||||
anchor.destroy()
|
||||
reset_context()
|
||||
|
||||
|
||||
def slots(pages, device="cpu"):
|
||||
return (
|
||||
torch.tensor(pages, device=device)[:, None] * PAGE
|
||||
+ torch.arange(PAGE, device=device)
|
||||
).reshape(-1)
|
||||
|
||||
|
||||
def compressed_slots(indices):
|
||||
return indices[::RATIO] // RATIO
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"pools",
|
||||
[
|
||||
("layer_first", "kernel"),
|
||||
("page_first", "kernel"),
|
||||
("layer_first", "direct"),
|
||||
("page_first_direct", "direct"),
|
||||
],
|
||||
indirect=True,
|
||||
)
|
||||
def test_qsa_host_roundtrip_and_consumer_fence(pools):
|
||||
device, anchor, host, backend = pools
|
||||
source = slots([3, 1], "cuda")
|
||||
destination = slots([4, 2], "cuda")
|
||||
host_indices = slots([2, 0])
|
||||
if backend == "kernel" and not host.can_use_write_back_jit:
|
||||
host_indices = host_indices.cuda()
|
||||
torch.manual_seed(42)
|
||||
device.qsa_compressed_flat.normal_()
|
||||
expected = [
|
||||
b[compressed_slots(source)].clone() for b in device.qsa_compressed_k_buffer_pool
|
||||
]
|
||||
engine = L2TransferEngine(backend)
|
||||
transfer = L2Transfer(
|
||||
host,
|
||||
device,
|
||||
host_indices,
|
||||
source.cpu() if backend == "direct" else source,
|
||||
{3: 0, 7: 1}.get,
|
||||
)
|
||||
engine.submit_device_to_host([transfer]).finish_event.synchronize()
|
||||
device.qsa_compressed_flat.fill_(-99)
|
||||
counter = LayerDoneCounter(8)
|
||||
producer = counter.update_producer()
|
||||
counter.set_consumer(producer)
|
||||
device.register_layer_transfer_counter(counter)
|
||||
with torch.cuda.stream(engine.host_to_device_stream):
|
||||
torch.cuda._sleep(20_000_000)
|
||||
engine.submit_host_to_device(
|
||||
[
|
||||
transfer._replace(
|
||||
host_indices=(
|
||||
host_indices.cuda() if backend == "kernel" else host_indices
|
||||
),
|
||||
device_indices=(
|
||||
destination.cpu() if backend == "direct" else destination
|
||||
),
|
||||
)
|
||||
],
|
||||
transfer_layer_id_max=8,
|
||||
on_layer_done=counter.events[producer].complete,
|
||||
)
|
||||
# Read through the production accessor without a global synchronize. The
|
||||
# indexer must wait for the stage-local layer event before reading its keys.
|
||||
actual = [
|
||||
device.get_qsa_compressed_k_buffer(layer)[compressed_slots(destination)]
|
||||
for layer in (7, 11)
|
||||
]
|
||||
for restored, saved in zip(actual, expected):
|
||||
torch.testing.assert_close(restored, saved, rtol=0, atol=0)
|
||||
assert host.size == anchor.size
|
||||
assert host.size_per_token * PAGE == sum(x[0].nbytes for x in host.device_buffers)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("pools", [("page_first_direct", "direct")], indirect=True)
|
||||
def test_qsa_file_restart_and_missing_sidecar(pools):
|
||||
device, anchor, host, backend = pools
|
||||
source, host_indices = slots([1, 3], "cuda"), slots([0, 2])
|
||||
device.qsa_compressed_flat.normal_()
|
||||
expected = [
|
||||
b[compressed_slots(source)].clone() for b in device.qsa_compressed_k_buffer_pool
|
||||
]
|
||||
engine = L2TransferEngine(backend)
|
||||
engine.submit_device_to_host(
|
||||
[L2Transfer(host, device, host_indices, source.cpu())]
|
||||
).finish_event.synchronize()
|
||||
config = HiCacheStorageConfig(
|
||||
tp_rank=0,
|
||||
tp_size=1,
|
||||
pp_rank=0,
|
||||
pp_size=1,
|
||||
attn_cp_rank=0,
|
||||
attn_cp_size=1,
|
||||
is_mla_model=False,
|
||||
enable_storage_metrics=False,
|
||||
is_page_first_layout=True,
|
||||
model_name="qsa-roundtrip",
|
||||
)
|
||||
keys = ["first", "second"]
|
||||
transfer = PoolTransfer(PoolName.INDEXER, host_indices=host_indices, keys=keys)
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
writer = HiCacheFile(config, file_path=directory)
|
||||
writer.register_mem_host_pool_v2(host, PoolName.INDEXER)
|
||||
for key in keys:
|
||||
assert writer.set(key, anchor.get_data_page(0))
|
||||
# Existing KV without compressed keys must never count as a usable hit.
|
||||
assert writer.batch_exists_v2(keys, [transfer]).kv_hit_pages == 0
|
||||
assert all(writer.batch_set_v2([transfer])[PoolName.INDEXER])
|
||||
reader = HiCacheFile(config, file_path=directory)
|
||||
reader.register_mem_host_pool_v2(host, PoolName.INDEXER)
|
||||
assert reader.batch_exists_v2(keys, [transfer]).kv_hit_pages == len(keys)
|
||||
host.kv_buffer.zero_()
|
||||
device.qsa_compressed_flat.fill_(-99)
|
||||
transfer.host_indices = slots([1, 3])
|
||||
assert all(reader.batch_get_v2([transfer])[PoolName.INDEXER])
|
||||
destination = slots([4, 2], "cuda")
|
||||
engine.submit_host_to_device(
|
||||
[L2Transfer(host, device, transfer.host_indices, destination.cpu())],
|
||||
transfer_layer_id_max=2,
|
||||
).finish_event.synchronize()
|
||||
for restored, saved in zip(device.qsa_compressed_k_buffer_pool, expected):
|
||||
torch.testing.assert_close(
|
||||
restored[compressed_slots(destination)], saved, rtol=0, atol=0
|
||||
)
|
||||
pointers, sizes = host.get_page_buffer_meta(transfer.host_indices)
|
||||
assert len(pointers) == len(keys)
|
||||
assert sizes == [host.get_data_page(0).nbytes] * len(keys)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("pools", [("page_first_direct", "direct")], indirect=True)
|
||||
def test_packed_mtp_indexer_roundtrip(pools):
|
||||
device, anchor, _, backend = pools
|
||||
draft = make_device_pool([0])
|
||||
host = QSAIndexerPoolHost(device, anchor, mtp_draft_device_pools=(draft,))
|
||||
try:
|
||||
source, destination, host_indices = slots([3, 1]), slots([4, 2]), slots([0, 2])
|
||||
all_buffers = [
|
||||
*device.qsa_compressed_k_buffer_pool,
|
||||
*draft.qsa_compressed_k_buffer_pool,
|
||||
]
|
||||
for buffer in all_buffers:
|
||||
buffer.normal_()
|
||||
expected = [b[compressed_slots(source).cuda()].clone() for b in all_buffers]
|
||||
engine = L2TransferEngine(backend)
|
||||
engine.submit_device_to_host(
|
||||
[L2Transfer(host, device, host_indices, source)]
|
||||
).finish_event.synchronize()
|
||||
# Whole-page storage payload must include the draft as well as target.
|
||||
pages = [host.get_data_page(i).clone() for i in (0, 2 * PAGE)]
|
||||
host.kv_buffer.zero_()
|
||||
for i, page in zip((0, 2 * PAGE), pages):
|
||||
host.set_from_flat_data_page(i, page)
|
||||
for buffer in all_buffers:
|
||||
buffer.fill_(-99)
|
||||
engine.submit_host_to_device(
|
||||
[
|
||||
L2Transfer(host, device, host_indices, destination, {3: 0, 7: 1}.get),
|
||||
L2Transfer(
|
||||
host, draft, host_indices, destination, {0: 2}.get, is_draft=True
|
||||
),
|
||||
],
|
||||
transfer_layer_id_max=8,
|
||||
).finish_event.synchronize()
|
||||
for buffer, saved in zip(all_buffers, expected):
|
||||
torch.testing.assert_close(
|
||||
buffer[compressed_slots(destination).cuda()], saved, rtol=0, atol=0
|
||||
)
|
||||
finally:
|
||||
host.destroy()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
import sys
|
||||
|
||||
sys.exit(pytest.main([__file__, "-v"]))
|
||||
@@ -19,14 +19,9 @@ from sglang.srt.layers.attention.qsa.kernel import (
|
||||
)
|
||||
from sglang.srt.layers.attention.qsa.qsa_indexer import QSAIndexer
|
||||
from sglang.srt.layers.rotary_embedding.mrope import MRotaryEmbedding
|
||||
|
||||
# MRotaryEmbedding reads the exec config bag at init; publish a minimal
|
||||
# process context for the bare pytest process.
|
||||
from sglang.srt.runtime_context import publish
|
||||
from sglang.srt.runtime_context import publish, reset_context
|
||||
from sglang.srt.server_args import ServerArgs
|
||||
|
||||
publish(ServerArgs(model_path="dummy"), role="test")
|
||||
|
||||
HEAD_DIM = 128
|
||||
NUM_Q_HEADS = 4
|
||||
RATIO = 4
|
||||
@@ -34,6 +29,15 @@ HIDDEN = 2560
|
||||
EPS = 1e-6
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def runtime_context():
|
||||
publish(ServerArgs(model_path="dummy"), role="test")
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
reset_context()
|
||||
|
||||
|
||||
def _make_config():
|
||||
return SimpleNamespace(
|
||||
indexer_n_heads=NUM_Q_HEADS,
|
||||
|
||||
Reference in New Issue
Block a user