[Env] centralize hicache vars in environ.py (#17204)
This commit is contained in:
@@ -268,6 +268,8 @@ class Envs:
|
|||||||
|
|
||||||
# Hi-Cache
|
# Hi-Cache
|
||||||
SGLANG_HICACHE_HF3FS_CONFIG_PATH = EnvStr(None)
|
SGLANG_HICACHE_HF3FS_CONFIG_PATH = EnvStr(None)
|
||||||
|
SGLANG_HICACHE_FILE_BACKEND_STORAGE_DIR = EnvStr(None)
|
||||||
|
SGLANG_HICACHE_NIXL_BACKEND_STORAGE_DIR = EnvStr(None)
|
||||||
|
|
||||||
# Mooncake KV Transfer
|
# Mooncake KV Transfer
|
||||||
SGLANG_MOONCAKE_CUSTOM_MEM_POOL = EnvStr(None)
|
SGLANG_MOONCAKE_CUSTOM_MEM_POOL = EnvStr(None)
|
||||||
|
|||||||
@@ -7,6 +7,7 @@ from typing import Any, List, Optional
|
|||||||
|
|
||||||
import torch
|
import torch
|
||||||
|
|
||||||
|
from sglang.srt.environ import envs
|
||||||
from sglang.srt.mem_cache.memory_pool_host import HostKVCache
|
from sglang.srt.mem_cache.memory_pool_host import HostKVCache
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
@@ -186,7 +187,7 @@ class HiCacheFile(HiCacheStorage):
|
|||||||
def __init__(
|
def __init__(
|
||||||
self, storage_config: HiCacheStorageConfig, file_path: str = "/tmp/hicache"
|
self, storage_config: HiCacheStorageConfig, file_path: str = "/tmp/hicache"
|
||||||
):
|
):
|
||||||
self.file_path = os.getenv("SGLANG_HICACHE_FILE_BACKEND_STORAGE_DIR", file_path)
|
self.file_path = envs.SGLANG_HICACHE_FILE_BACKEND_STORAGE_DIR.get() or file_path
|
||||||
|
|
||||||
tp_rank, tp_size, model_name, is_mla_model = (
|
tp_rank, tp_size, model_name, is_mla_model = (
|
||||||
storage_config.tp_rank,
|
storage_config.tp_rank,
|
||||||
|
|||||||
@@ -1,11 +1,11 @@
|
|||||||
import logging
|
import logging
|
||||||
import os
|
|
||||||
import time
|
import time
|
||||||
import uuid
|
import uuid
|
||||||
from typing import Any, List, Optional, Union
|
from typing import Any, List, Optional, Union
|
||||||
|
|
||||||
import torch
|
import torch
|
||||||
|
|
||||||
|
from sglang.srt.environ import envs
|
||||||
from sglang.srt.mem_cache.hicache_storage import HiCacheStorage, HiCacheStorageConfig
|
from sglang.srt.mem_cache.hicache_storage import HiCacheStorage, HiCacheStorageConfig
|
||||||
|
|
||||||
from .nixl_utils import (
|
from .nixl_utils import (
|
||||||
@@ -44,7 +44,7 @@ class HiCacheNixl(HiCacheStorage):
|
|||||||
plugin = nixlconfig.get_specified_plugin()
|
plugin = nixlconfig.get_specified_plugin()
|
||||||
|
|
||||||
# Might be better to be unified across HiCache backends and moved to HiCacheController
|
# Might be better to be unified across HiCache backends and moved to HiCacheController
|
||||||
file_path = os.getenv("SGLANG_HICACHE_NIXL_BACKEND_STORAGE_DIR", file_path)
|
file_path = envs.SGLANG_HICACHE_NIXL_BACKEND_STORAGE_DIR.get() or file_path
|
||||||
self.file_manager = (
|
self.file_manager = (
|
||||||
NixlFileManager(file_path)
|
NixlFileManager(file_path)
|
||||||
if plugin not in NixlBackendSelection.OBJ_PLUGINS
|
if plugin not in NixlBackendSelection.OBJ_PLUGINS
|
||||||
|
|||||||
Reference in New Issue
Block a user