[Feat] unified_cache: Support FULL + MAMBA MLA models (Kimi-K3) in the direct external linker (#44977)
Co-authored-by: Yutong Wu <yutong.wu@amd.com> Co-authored-by: Zhangheng <hzh0425@apache.org>
This commit is contained in:
co-authored by
Yutong Wu
Zhangheng
parent
f1ba723c64
commit
eb91a41f38
@@ -72,6 +72,8 @@ class CacheInitParams:
|
||||
tree_core_backend: Optional[str] = dataclasses.field(default=None, kw_only=True)
|
||||
|
||||
mtp_draft_device_pools: tuple[object, ...] = ()
|
||||
# Draft KV pools that share the target KV slots but are stored apart from it.
|
||||
sidecar_draft_device_pools: tuple[object, ...] = ()
|
||||
mtp_draft_mamba_pools: tuple[object, ...] = ()
|
||||
|
||||
@property
|
||||
|
||||
@@ -1960,6 +1960,13 @@ class _MambaStrategy(StackStrategy):
|
||||
ComponentType.MAMBA,
|
||||
}
|
||||
|
||||
def build_direct_linker_pool_group(self, *, kvcache, params, page_size):
|
||||
from sglang.srt.mem_cache.hybrid_cache.linker_pool_assembler import (
|
||||
_build_mamba_device_pool_group,
|
||||
)
|
||||
|
||||
return _build_mamba_device_pool_group(kvcache, params, page_size)
|
||||
|
||||
def build(
|
||||
self,
|
||||
*,
|
||||
|
||||
@@ -14,7 +14,7 @@ from sglang.srt.mem_cache.hicache_storage import (
|
||||
PoolTransfer,
|
||||
)
|
||||
from sglang.srt.mem_cache.unified_cache.component_type import ComponentType
|
||||
from sglang.srt.runtime_context import get_memory
|
||||
from sglang.srt.runtime_context import get_memory, get_parallel
|
||||
|
||||
|
||||
class DevicePoolEntry:
|
||||
@@ -460,7 +460,11 @@ def _build_dsa_device_pool_group(
|
||||
|
||||
|
||||
def _mha_device_pool_entry(
|
||||
name: PoolName, pool: Any, layer_mapping: dict[int, int], page_size: int
|
||||
name: PoolName,
|
||||
pool: Any,
|
||||
layer_mapping: dict[int, int],
|
||||
page_size: int,
|
||||
indices_from_pool: PoolName | None = None,
|
||||
) -> DevicePoolEntry:
|
||||
# TODO(alphabetc1): support mxfp8.
|
||||
if pool.k_scale_buffer is not None or pool.native_k_scale_buffer is not None:
|
||||
@@ -470,7 +474,7 @@ def _mha_device_pool_entry(
|
||||
num_layers = len(pool.k_buffer)
|
||||
return DevicePoolEntry(
|
||||
name=name,
|
||||
indices_from_pool=name,
|
||||
indices_from_pool=indices_from_pool or name,
|
||||
device_pool=pool,
|
||||
components=[[*pool.k_buffer, *pool.v_buffer]],
|
||||
layer_mapping={
|
||||
@@ -488,6 +492,23 @@ def _mamba_device_pool_entry(
|
||||
_stage_local_layer_mapping,
|
||||
)
|
||||
|
||||
# TODO(alphabetc1): support unified memory for hybrid Mamba models.
|
||||
if get_memory().enable_unified_memory:
|
||||
raise ValueError(
|
||||
"The direct external linker does not support --enable-unified-memory "
|
||||
"on hybrid Mamba models yet."
|
||||
)
|
||||
# TODO(alphabetc1): support int8 Mamba checkpoints and Mamba side states.
|
||||
if (
|
||||
req_to_token_pool.mamba_ckpt_pool is not None
|
||||
or req_to_token_pool.short_conv_pool.enabled
|
||||
or req_to_token_pool.ngram_pool.enabled
|
||||
):
|
||||
raise ValueError(
|
||||
"The direct external linker does not support int8 Mamba checkpoints "
|
||||
"or Mamba side states yet."
|
||||
)
|
||||
|
||||
mamba_pool = req_to_token_pool.mamba_pool
|
||||
mamba_cache = mamba_pool.mamba_cache
|
||||
layers = range(mamba_pool.num_mamba_layers)
|
||||
@@ -517,24 +538,6 @@ def _build_mamba_swa_device_pool_group(
|
||||
_swa_layer_mappings,
|
||||
)
|
||||
|
||||
# TODO(alphabetc1): support unified memory for hybrid Mamba models.
|
||||
if get_memory().enable_unified_memory:
|
||||
raise ValueError(
|
||||
"The direct external linker does not support --enable-unified-memory "
|
||||
"on hybrid Mamba models yet."
|
||||
)
|
||||
# TODO(alphabetc1): support int8 Mamba checkpoints and Mamba side states.
|
||||
req_to_token_pool = params.req_to_token_pool
|
||||
if (
|
||||
req_to_token_pool.mamba_ckpt_pool is not None
|
||||
or req_to_token_pool.short_conv_pool.enabled
|
||||
or req_to_token_pool.ngram_pool.enabled
|
||||
):
|
||||
raise ValueError(
|
||||
"The direct external linker does not support int8 Mamba checkpoints "
|
||||
"or Mamba side states yet."
|
||||
)
|
||||
|
||||
full_mapping, swa_mapping = _swa_layer_mappings(kvcache)
|
||||
entries = [
|
||||
_mha_device_pool_entry(
|
||||
@@ -549,6 +552,63 @@ def _build_mamba_swa_device_pool_group(
|
||||
return DevicePoolGroup(entries, num_layers + 1, page_size)
|
||||
|
||||
|
||||
def _build_mamba_device_pool_group(
|
||||
kvcache: Any, params: Any, page_size: int
|
||||
) -> DevicePoolGroup:
|
||||
from sglang.srt.mem_cache.hybrid_cache.hybrid_pool_assembler import (
|
||||
_stage_local_layer_mapping,
|
||||
)
|
||||
from sglang.srt.mem_cache.memory_pool import (
|
||||
HybridLinearKVPool,
|
||||
MLATokenToKVPool,
|
||||
)
|
||||
|
||||
# TODO(alphabetc1): support other full-attention pools, DCP and packed draft KV.
|
||||
full_kv_pool = kvcache.full_kv_pool
|
||||
if (
|
||||
type(kvcache) is not HybridLinearKVPool
|
||||
or type(full_kv_pool) is not MLATokenToKVPool
|
||||
or get_parallel().attn_dcp_size > 1
|
||||
or params.mtp_draft_device_pools
|
||||
):
|
||||
raise ValueError(
|
||||
"The direct external linker supports FULL + MAMBA models only with a "
|
||||
"plain MLA pool, without DCP or packed draft KV."
|
||||
)
|
||||
|
||||
entries = [
|
||||
DevicePoolEntry(
|
||||
name=PoolName.KV,
|
||||
indices_from_pool=PoolName.KV,
|
||||
device_pool=full_kv_pool,
|
||||
components=[full_kv_pool.kv_buffer],
|
||||
layer_mapping=_stage_local_layer_mapping(
|
||||
kvcache.full_attention_layer_id_mapping, kvcache.start_layer
|
||||
),
|
||||
page_size=page_size,
|
||||
rows_are_pages=False,
|
||||
),
|
||||
_mamba_device_pool_entry(params.req_to_token_pool, kvcache.start_layer),
|
||||
]
|
||||
for draft_pool in params.sidecar_draft_device_pools:
|
||||
if draft_pool.page_size != page_size:
|
||||
raise ValueError("Draft KV page size must match the tree page size.")
|
||||
# Draft layer i rides transfer layer i; the draft reads it only after
|
||||
# the batch's loads finish.
|
||||
entries.append(
|
||||
_mha_device_pool_entry(
|
||||
PoolName.DRAFT,
|
||||
draft_pool,
|
||||
{layer: layer for layer in range(draft_pool.layer_num)},
|
||||
page_size,
|
||||
indices_from_pool=PoolName.KV,
|
||||
)
|
||||
)
|
||||
num_layers = max(layer for entry in entries for layer in entry.layer_mapping)
|
||||
# Not rank_replicated: the Mamba state is TP-sharded.
|
||||
return DevicePoolGroup(entries, num_layers + 1, page_size)
|
||||
|
||||
|
||||
def resolve_hybrid_device_pool_group(
|
||||
*,
|
||||
kvcache: Any,
|
||||
@@ -561,8 +621,15 @@ def resolve_hybrid_device_pool_group(
|
||||
_select_strategy,
|
||||
)
|
||||
|
||||
return _select_strategy(kvcache, components).build_direct_linker_pool_group(
|
||||
group = _select_strategy(kvcache, components).build_direct_linker_pool_group(
|
||||
kvcache=kvcache,
|
||||
params=params,
|
||||
page_size=page_size,
|
||||
)
|
||||
if getattr(params, "sidecar_draft_device_pools", ()) and (
|
||||
PoolName.DRAFT not in group.entry_map
|
||||
):
|
||||
raise ValueError(
|
||||
"The direct external linker does not support this model's draft KV."
|
||||
)
|
||||
return group
|
||||
|
||||
@@ -409,6 +409,12 @@ def build_kv_cache(
|
||||
draft_swa_window=draft_swa_window,
|
||||
mtp_draft_device_pools=mtp_draft_device_pools,
|
||||
mtp_draft_mamba_pools=tp_worker.model_runner.mtp_draft_mamba_pools,
|
||||
sidecar_draft_device_pools=(
|
||||
hicache_draft_plan.device_pools
|
||||
if hicache_draft_plan is not None
|
||||
and hicache_draft_plan.mode == HiCacheDraftMode.SIDECAR
|
||||
else ()
|
||||
),
|
||||
)
|
||||
if params.swa_retain_window != sliding_window_size:
|
||||
logger.info(
|
||||
|
||||
@@ -307,9 +307,9 @@ class BaseSpecWorker(ABC):
|
||||
device_pools=draft_pools,
|
||||
)
|
||||
|
||||
if get_memory().enable_unified_cache_external_linker:
|
||||
if get_memory().enable_unified_cache_external_linker and len(draft_pools) > 1:
|
||||
raise NotImplementedError(
|
||||
"The external linker only supports packed draft KV caches."
|
||||
"The external linker supports one non-packed draft KV pool."
|
||||
)
|
||||
|
||||
return HiCacheDraftPlan(
|
||||
|
||||
+115
@@ -0,0 +1,115 @@
|
||||
"""Mooncake direct linker on Kimi Linear (MLA + KDA, as Kimi-K3): a host hit decodes
|
||||
bit-for-bit like a device hit."""
|
||||
|
||||
import unittest
|
||||
|
||||
from sglang.test.ci.ci_register import register_cuda_ci
|
||||
from sglang.test.kits.unified_radix_cache_kit import _random_suffixes
|
||||
from sglang.test.kl_test_utils import (
|
||||
_extract_output_logprobs,
|
||||
_flush_cache,
|
||||
_generate,
|
||||
get_input_ids,
|
||||
)
|
||||
from sglang.test.mooncake_utils import MooncakeTestServices
|
||||
from sglang.test.test_utils import (
|
||||
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
|
||||
CustomTestCase,
|
||||
find_available_port,
|
||||
popen_launch_server,
|
||||
terminate_and_kill_process_tree,
|
||||
unified_radix_tree_server_env,
|
||||
)
|
||||
|
||||
register_cuda_ci(
|
||||
est_time=180,
|
||||
stage="extra-b",
|
||||
# Direct GPU transfers require an RDMA-capable runner, even for TP=1.
|
||||
runner_config="4-gpu-h100",
|
||||
)
|
||||
|
||||
_MODEL_PATH = "yujiepan/kimi-linear-tiny-random"
|
||||
|
||||
PAGE_SIZE = 64
|
||||
TRACK_INTERVAL = 128
|
||||
# A checkpoint boundary; requests end inside the next page so both hits stop here.
|
||||
PREFIX_LEN = 1024
|
||||
|
||||
|
||||
class TestKimiLinearUnifiedCacheLinker(CustomTestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
cls.model = _MODEL_PATH
|
||||
cls.base_url = f"http://127.0.0.1:{find_available_port(30000)}"
|
||||
cls.mooncake = MooncakeTestServices()
|
||||
cls.addClassCleanup(cls.mooncake.stop)
|
||||
cls.mooncake.start()
|
||||
other_args = [
|
||||
"--trust-remote-code",
|
||||
"--attention-backend",
|
||||
"triton",
|
||||
"--linear-attn-backend",
|
||||
"triton",
|
||||
"--mamba-backend",
|
||||
"triton",
|
||||
"--page-size",
|
||||
str(PAGE_SIZE),
|
||||
"--mamba-radix-cache-strategy",
|
||||
"extra_buffer",
|
||||
"--mamba-track-interval",
|
||||
str(TRACK_INTERVAL),
|
||||
"--enable-deterministic-inference",
|
||||
"--mem-fraction-static",
|
||||
"0.6",
|
||||
"--max-running-requests",
|
||||
"1",
|
||||
"--enable-cache-report",
|
||||
"--enable-unified-cache-external-linker",
|
||||
]
|
||||
cls.process = popen_launch_server(
|
||||
cls.model,
|
||||
cls.base_url,
|
||||
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
|
||||
other_args=other_args,
|
||||
env=unified_radix_tree_server_env("python", **cls.mooncake.server_env()),
|
||||
)
|
||||
cls.addClassCleanup(
|
||||
terminate_and_kill_process_tree, cls.process, wait_timeout=60
|
||||
)
|
||||
cls.input_ids = get_input_ids(
|
||||
tokenizer_path=cls.model, num_samples=4, trust_remote_code=True
|
||||
)
|
||||
|
||||
def test_host_hit_matches_device_hit(self):
|
||||
prompts = [list(ids[: PREFIX_LEN + 1]) for ids in self.input_ids]
|
||||
for ids in prompts:
|
||||
_generate(self.base_url, [ids], 1)
|
||||
requests = [
|
||||
prompt + suffix
|
||||
for prompt, suffix in zip(
|
||||
prompts, _random_suffixes(len(prompts), PAGE_SIZE - 1, seed=100)
|
||||
)
|
||||
]
|
||||
|
||||
run = lambda: [
|
||||
_generate(self.base_url, [ids], 32, return_logprob=True)[0]
|
||||
for ids in requests
|
||||
]
|
||||
device_results = run()
|
||||
_flush_cache(self.base_url)
|
||||
host_results = run()
|
||||
|
||||
for device, host in zip(device_results, host_results):
|
||||
for result, host_tokens in ((device, 0), (host, PREFIX_LEN)):
|
||||
meta = result["meta_info"]
|
||||
details = meta.get("cached_tokens_details") or {}
|
||||
self.assertEqual(meta["cached_tokens"], PREFIX_LEN)
|
||||
self.assertEqual(int(details.get("host", 0)), host_tokens)
|
||||
self.assertEqual(host["output_ids"], device["output_ids"])
|
||||
self.assertEqual(
|
||||
_extract_output_logprobs(host), _extract_output_logprobs(device)
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -645,6 +645,15 @@ class TestHybridDevicePoolAssembler(CustomTestCase):
|
||||
params=SimpleNamespace(mtp_draft_device_pools=draft_pools),
|
||||
components={ComponentType.FULL},
|
||||
)
|
||||
with self.assertRaisesRegex(ValueError, "draft KV"):
|
||||
resolve_hybrid_device_pool_group(
|
||||
kvcache=kvcache,
|
||||
page_size=2,
|
||||
params=SimpleNamespace(
|
||||
mtp_draft_device_pools=(), sidecar_draft_device_pools=(object(),)
|
||||
),
|
||||
components={ComponentType.FULL},
|
||||
)
|
||||
|
||||
self.assertEqual(group.num_layers, 2)
|
||||
self.assertTrue(group.rank_replicated)
|
||||
@@ -672,8 +681,7 @@ class TestHybridDevicePoolAssembler(CustomTestCase):
|
||||
self.assertEqual(sizes, [[11, 23]])
|
||||
self.assertEqual(offsets, [[7, 35]])
|
||||
|
||||
def test_linker_requires_packed_draft(self):
|
||||
"""Do not accept draft state that the linker would omit from storage."""
|
||||
def test_draft_plan_packs_only_nextn_mtp(self):
|
||||
from sglang.srt.speculative import base_spec_worker as spec
|
||||
from sglang.srt.speculative.spec_info import SpeculativeAlgorithm
|
||||
|
||||
@@ -710,39 +718,109 @@ class TestHybridDevicePoolAssembler(CustomTestCase):
|
||||
),
|
||||
),
|
||||
):
|
||||
if linker_enabled and not nextn_layers:
|
||||
with self.assertRaisesRegex(
|
||||
NotImplementedError, "only supports packed"
|
||||
):
|
||||
spec.BaseSpecWorker._build_hicache_draft_plan(worker)
|
||||
self.assertEqual(target.mtp_draft_device_pools, ())
|
||||
else:
|
||||
plan = spec.BaseSpecWorker._build_hicache_draft_plan(worker)
|
||||
self.assertEqual(
|
||||
plan.mode,
|
||||
spec.HiCacheDraftMode.PACKED
|
||||
if nextn_layers
|
||||
else spec.HiCacheDraftMode.SIDECAR,
|
||||
)
|
||||
self.assertEqual(plan.device_pools, (draft.token_to_kv_pool,))
|
||||
self.assertEqual(
|
||||
target.mtp_draft_device_pools,
|
||||
plan.device_pools if nextn_layers else (),
|
||||
)
|
||||
plan = spec.BaseSpecWorker._build_hicache_draft_plan(worker)
|
||||
self.assertEqual(
|
||||
plan.mode,
|
||||
spec.HiCacheDraftMode.PACKED
|
||||
if nextn_layers
|
||||
else spec.HiCacheDraftMode.SIDECAR,
|
||||
)
|
||||
self.assertEqual(plan.device_pools, (draft.token_to_kv_pool,))
|
||||
self.assertEqual(
|
||||
target.mtp_draft_device_pools,
|
||||
plan.device_pools if nextn_layers else (),
|
||||
)
|
||||
|
||||
def test_unsupported_strategy_fails_with_context(self):
|
||||
from sglang.srt.mem_cache.memory_pool import HybridLinearKVPool
|
||||
def test_mla_mamba_maps_kv_pages_and_state_slots(self):
|
||||
from sglang.srt.mem_cache.hybrid_cache import linker_pool_assembler
|
||||
from sglang.srt.mem_cache.memory_pool import (
|
||||
HybridLinearKVPool,
|
||||
MHATokenToKVPool,
|
||||
MLATokenToKVPool,
|
||||
)
|
||||
|
||||
kvcache = HybridLinearKVPool.__new__(HybridLinearKVPool)
|
||||
kvcache.start_layer = 0
|
||||
kvcache.full_attention_layer_id_mapping = {1: 0, 3: 1}
|
||||
kvcache.full_kv_pool = MLATokenToKVPool.__new__(MLATokenToKVPool)
|
||||
kvcache.full_kv_pool.kv_buffer = [
|
||||
torch.zeros((8, 1, 5), dtype=torch.uint8) for _ in range(2)
|
||||
]
|
||||
mamba_cache = SimpleNamespace(
|
||||
temporal=torch.zeros((2, 4, 3), dtype=torch.uint8),
|
||||
conv=[torch.zeros((2, 4, 2), dtype=torch.uint8)],
|
||||
)
|
||||
draft = MHATokenToKVPool.__new__(MHATokenToKVPool)
|
||||
draft.page_size, draft.layer_num = 2, 1
|
||||
draft.k_scale_buffer = draft.native_k_scale_buffer = None
|
||||
draft.k_buffer = [torch.zeros((8, 3), dtype=torch.uint8)]
|
||||
draft.v_buffer = [torch.zeros((8, 3), dtype=torch.uint8)]
|
||||
params = SimpleNamespace(
|
||||
mtp_draft_device_pools=(),
|
||||
sidecar_draft_device_pools=(draft,),
|
||||
req_to_token_pool=SimpleNamespace(
|
||||
mamba_pool=SimpleNamespace(mamba_cache=mamba_cache, num_mamba_layers=2),
|
||||
mamba_map={0: 0, 2: 1},
|
||||
mamba_ckpt_pool=None,
|
||||
short_conv_pool=SimpleNamespace(enabled=False),
|
||||
ngram_pool=SimpleNamespace(enabled=False),
|
||||
),
|
||||
)
|
||||
|
||||
def resolve(dcp_size=1):
|
||||
with (
|
||||
patch.object(
|
||||
linker_pool_assembler,
|
||||
"get_memory",
|
||||
return_value=SimpleNamespace(enable_unified_memory=False),
|
||||
),
|
||||
patch.object(
|
||||
linker_pool_assembler,
|
||||
"get_parallel",
|
||||
return_value=SimpleNamespace(attn_dcp_size=dcp_size),
|
||||
),
|
||||
):
|
||||
return resolve_hybrid_device_pool_group(
|
||||
kvcache=kvcache,
|
||||
page_size=2,
|
||||
params=params,
|
||||
components={ComponentType.FULL, ComponentType.MAMBA},
|
||||
)
|
||||
|
||||
group = resolve()
|
||||
self.assertEqual(group.num_layers, 4)
|
||||
self.assertFalse(group.rank_replicated)
|
||||
kv, mamba = group.entry_map[PoolName.KV], group.entry_map[PoolName.MAMBA]
|
||||
self.assertEqual(kv.layer_mapping, {1: 0, 3: 1})
|
||||
self.assertEqual(mamba.layer_mapping, {0: 0, 2: 1})
|
||||
self.assertEqual(kv.prepare_locations(torch.tensor([2, 3])), [2])
|
||||
self.assertEqual(mamba.prepare_locations(torch.tensor([3])), [3])
|
||||
self.assertEqual(group.sources[PoolName.DRAFT], PoolName.KV)
|
||||
self.assertEqual(group.entry_map[PoolName.DRAFT].layer_mapping, {0: (0, 1)})
|
||||
|
||||
with self.assertRaisesRegex(ValueError, "plain MLA"):
|
||||
resolve(dcp_size=2)
|
||||
params.mtp_draft_device_pools = (object(),)
|
||||
with self.assertRaisesRegex(ValueError, "plain MLA"):
|
||||
resolve()
|
||||
params.mtp_draft_device_pools = ()
|
||||
kvcache.full_kv_pool = MHATokenToKVPool.__new__(MHATokenToKVPool)
|
||||
with self.assertRaisesRegex(ValueError, "plain MLA"):
|
||||
resolve()
|
||||
|
||||
def test_unsupported_strategy_fails_with_context(self):
|
||||
from sglang.srt.mem_cache.swa_memory_pool import SWAKVPool
|
||||
|
||||
kvcache = SWAKVPool.__new__(SWAKVPool)
|
||||
with self.assertRaisesRegex(
|
||||
ValueError,
|
||||
"does not support the direct external linker: _MambaStrategy",
|
||||
"does not support the direct external linker: _SwaStrategy",
|
||||
):
|
||||
resolve_hybrid_device_pool_group(
|
||||
kvcache=kvcache,
|
||||
page_size=2,
|
||||
params=SimpleNamespace(),
|
||||
components={ComponentType.FULL, ComponentType.MAMBA},
|
||||
components={ComponentType.FULL, ComponentType.SWA},
|
||||
)
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user