[HiCache] Support DCP with DSpark (#35221)
Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
@@ -1013,9 +1013,11 @@ def build_full_draft_pools(
|
||||
controller = tree_cache.cache_controller
|
||||
host_pool_group = controller.mem_pool_host
|
||||
|
||||
# Note(kpham-sgl): DCP x DSpark draft KV is replicated and spans the virtual
|
||||
# loc space, so match the target host's logical_size instead of physical size.
|
||||
draft_host_pool = _build_mha_mla_host_pool(
|
||||
pool=pool,
|
||||
host_to_device_ratio=host_pool_group.size / pool.size,
|
||||
host_to_device_ratio=host_pool_group.logical_size / pool.size,
|
||||
page_size=controller.page_size,
|
||||
layout=server_args.hicache_mem_layout,
|
||||
allocator_type=_get_allocator_type(server_args),
|
||||
|
||||
@@ -117,7 +117,7 @@ def _register_legacy_hicache_draft(
|
||||
# so that host indices stay 1-to-1 between target and draft KV caches.
|
||||
primary_host_pool = tree_cache.cache_controller.mem_pool_host
|
||||
host_pool_kwargs = dict(
|
||||
host_to_device_ratio=primary_host_pool.size / pool.size,
|
||||
host_to_device_ratio=primary_host_pool.logical_size / pool.size,
|
||||
host_size=0,
|
||||
page_size=page_size,
|
||||
layout=server_args.hicache_mem_layout,
|
||||
|
||||
@@ -7502,11 +7502,11 @@ class ServerArgs:
|
||||
"backup and the storage keys must become dcp_rank-aware "
|
||||
"first. Run HiCache+DCP with L1/L2 only."
|
||||
)
|
||||
if self.speculative_algorithm is not None:
|
||||
if self.speculative_algorithm not in (None, "DSPARK"):
|
||||
raise NotImplementedError(
|
||||
"HiCache with --dcp-size > 1 does not support speculative "
|
||||
"decoding yet (the draft-model host pool has no DCP index "
|
||||
"translation)."
|
||||
"HiCache with --dcp-size > 1 only supports DSPARK speculative "
|
||||
"decoding; other draft-model host pools have no DCP index "
|
||||
"translation."
|
||||
)
|
||||
if self.enable_lmcache:
|
||||
raise NotImplementedError(
|
||||
|
||||
Reference in New Issue
Block a user