[Performance] Decode Offload improves the long texts performance 100% through dynamic block offload. (#17216)
Co-authored-by: zhangheng <hzh0425@apache.org>
This commit is contained in:
@@ -576,6 +576,8 @@ class HiCacheController:
|
||||
model_name: Optional[str] = None,
|
||||
storage_backend_extra_config: Optional[dict] = None,
|
||||
):
|
||||
if storage_backend_extra_config is None:
|
||||
storage_backend_extra_config = {}
|
||||
|
||||
if is_dp_attention_enabled():
|
||||
self.tp_rank = get_attention_tp_rank()
|
||||
|
||||
Reference in New Issue
Block a user