[HiCache] Add CP support for HiCache (#20977)
Signed-off-by: Shangming Cai <csmthu@gmail.com>
This commit is contained in:
@@ -261,6 +261,8 @@ class HiCacheController:
|
||||
storage_backend_extra_config: Optional[dict] = None,
|
||||
pp_rank: int = 0,
|
||||
pp_size: int = 1,
|
||||
attn_cp_rank: int = 0,
|
||||
attn_cp_size: int = 1,
|
||||
enable_storage_metrics: bool = False,
|
||||
):
|
||||
self.tp_group = tp_group
|
||||
@@ -280,6 +282,8 @@ class HiCacheController:
|
||||
self.storage_backend_type = None
|
||||
self.pp_rank = pp_rank
|
||||
self.pp_size = pp_size
|
||||
self.attn_cp_rank = attn_cp_rank
|
||||
self.attn_cp_size = attn_cp_size
|
||||
self.enable_storage_metrics = enable_storage_metrics
|
||||
|
||||
# Default storage page IO functions (may be overridden by attach).
|
||||
@@ -611,6 +615,8 @@ class HiCacheController:
|
||||
tp_size=self.tp_size,
|
||||
pp_rank=self.pp_rank,
|
||||
pp_size=self.pp_size,
|
||||
attn_cp_rank=self.attn_cp_rank,
|
||||
attn_cp_size=self.attn_cp_size,
|
||||
is_mla_model=is_mla_backend,
|
||||
enable_storage_metrics=self.enable_storage_metrics,
|
||||
is_page_first_layout=self.mem_pool_host.layout == "page_first",
|
||||
|
||||
Reference in New Issue
Block a user