[HiCache] Route --file-storage-path to the file storage backend (#33883)

Signed-off-by: adlashab <adlashab@amd.com>
This commit is contained in:
ADiL
2026-10-05 15:33:49 -07:00
committed by GitHub
parent 61ba98443d
commit 968726f3ce
4 changed files with 103 additions and 3 deletions
@@ -183,9 +183,9 @@ class Serving(msgspec.Struct):
"The buliltin completion template name or the path of the completion template file. This is only used for OpenAI-compatible API server. only for code completion currently.",
] = None
file_storage_path: A[
str,
Optional[str],
"The path of the file storage in backend.",
] = "sglang_storage"
] = None
enable_cache_report: A[
bool,
"Return number of cached tokens in usage.prompt_tokens_details for each openai request.",
@@ -382,7 +382,11 @@ class HiCacheFile(HiCacheStorage):
def __init__(
self, storage_config: HiCacheStorageConfig, file_path: str = "/tmp/hicache"
):
self.file_path = envs.SGLANG_HICACHE_FILE_BACKEND_STORAGE_DIR.get() or file_path
self.file_path = (
envs.SGLANG_HICACHE_FILE_BACKEND_STORAGE_DIR.get()
or (storage_config.extra_config or {}).get("file_storage_path")
or file_path
)
tp_rank, tp_size, pp_rank, pp_size, model_name, is_mla_model = (
storage_config.tp_rank,
@@ -442,6 +442,11 @@ class UnifiedRadixCache(BasePrefixCache):
) = HybridCacheController.parse_storage_backend_extra_config(
get_memory().hicache_storage_backend_extra_config
)
if server_args.file_storage_path:
storage_extra_config = dict(storage_extra_config or {})
storage_extra_config.setdefault(
"file_storage_path", server_args.file_storage_path
)
storage_prefetch_threshold = (
self._storage_attachment.resolve_prefetch_threshold(
storage_prefetch_threshold