[HiCache] Route --file-storage-path to the file storage backend (#33883)
Signed-off-by: adlashab <adlashab@amd.com>
This commit is contained in:
@@ -183,9 +183,9 @@ class Serving(msgspec.Struct):
|
||||
"The buliltin completion template name or the path of the completion template file. This is only used for OpenAI-compatible API server. only for code completion currently.",
|
||||
] = None
|
||||
file_storage_path: A[
|
||||
str,
|
||||
Optional[str],
|
||||
"The path of the file storage in backend.",
|
||||
] = "sglang_storage"
|
||||
] = None
|
||||
enable_cache_report: A[
|
||||
bool,
|
||||
"Return number of cached tokens in usage.prompt_tokens_details for each openai request.",
|
||||
|
||||
@@ -382,7 +382,11 @@ class HiCacheFile(HiCacheStorage):
|
||||
def __init__(
|
||||
self, storage_config: HiCacheStorageConfig, file_path: str = "/tmp/hicache"
|
||||
):
|
||||
self.file_path = envs.SGLANG_HICACHE_FILE_BACKEND_STORAGE_DIR.get() or file_path
|
||||
self.file_path = (
|
||||
envs.SGLANG_HICACHE_FILE_BACKEND_STORAGE_DIR.get()
|
||||
or (storage_config.extra_config or {}).get("file_storage_path")
|
||||
or file_path
|
||||
)
|
||||
|
||||
tp_rank, tp_size, pp_rank, pp_size, model_name, is_mla_model = (
|
||||
storage_config.tp_rank,
|
||||
|
||||
@@ -442,6 +442,11 @@ class UnifiedRadixCache(BasePrefixCache):
|
||||
) = HybridCacheController.parse_storage_backend_extra_config(
|
||||
get_memory().hicache_storage_backend_extra_config
|
||||
)
|
||||
if server_args.file_storage_path:
|
||||
storage_extra_config = dict(storage_extra_config or {})
|
||||
storage_extra_config.setdefault(
|
||||
"file_storage_path", server_args.file_storage_path
|
||||
)
|
||||
storage_prefetch_threshold = (
|
||||
self._storage_attachment.resolve_prefetch_threshold(
|
||||
storage_prefetch_threshold
|
||||
|
||||
Reference in New Issue
Block a user