minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

Add Mooncake tenant id support (#30256)

Browse Source
This commit is contained in:
LZW
2026-07-29 18:17:41 +08:00
committed by GitHub
parent 8742a1a0f8
commit 4c82bb3252
5 changed files with 276 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python/sglang/srt/environ.py
+1
View File
@@ -524,6 +524,7 @@ class Envs:
MOONCAKE_STANDALONE_STORAGE = EnvBool(False)
MOONCAKE_ENABLE_SSD_OFFLOAD = EnvBool(False)
MOONCAKE_OFFLOAD_FILE_STORAGE_PATH = EnvStr(None)
MOONCAKE_TENANT_ID = EnvStr("default")
# MoRI KV Transfer
# Send CPU-resident AUX data via RDMA instead of ZMQ TCP (default: TCP).
python/sglang/srt/mem_cache/storage/mooncake_store/README.md
+12 -2
View File
@@ -203,7 +203,7 @@ python -m sglang.launch_server \
--enable-hierarchical-cache \
--hicache-storage-backend mooncake \
--model-path [model_path] \
--hicache-storage-backend-extra-config '{"master_server_address": "127.0.0.1:50051", "local_hostname": "localhost", "metadata_server": "http://127.0.0.1:8080/metadata", "global_segment_size": "4gb", "protocol": "rdma", "device_name": ""}'
--hicache-storage-backend-extra-config '{"master_server_address": "127.0.0.1:50051", "local_hostname": "localhost", "metadata_server": "http://127.0.0.1:8080/metadata", "global_segment_size": "4gb", "protocol": "rdma", "device_name": "", "tenant_id": "tenant-a"}'
```
**Using JSON file to configure Mooncake**
@@ -219,7 +219,8 @@ echo '{
"master_server_address": "127.0.0.1:50051",
"protocol": "rdma",
"device_name": "",
"global_segment_size": "4gb"
"global_segment_size": "4gb",
"tenant_id": "tenant-a"
}' > ${SGLANG_HICACHE_MOONCAKE_CONFIG_PATH}
python -m sglang.launch_server \
@@ -236,6 +237,7 @@ MOONCAKE_MASTER="127.0.0.1:50051" \
MOONCAKE_PROTOCOL="rdma" \
MOONCAKE_DEVICE="" \
MOONCAKE_GLOBAL_SEGMENT_SIZE="4gb" \
MOONCAKE_TENANT_ID="tenant-a" \
python -m sglang.launch_server \
--enable-hierarchical-cache \
--hicache-storage-backend mooncake\
@@ -250,6 +252,14 @@ In particular, for the `global segment size`, if at least one `store service` in
**Important:** when `tp > 1`, each Tensor Parallel (TP) rank launches its own Mooncake backend instance and contributes `1/global_segment_size` memory. Therefore, the total memory consumption equals `global segment size`.
**Tenant Isolation (`tenant_id`):**
When `tenant_id` is set, SGLang forwards it to `MooncakeDistributedStore.setup(..., tenant_id=...)`. Producers and consumers that should share HiCache data must use the same `tenant_id`.
You can configure it through `tenant_id` in `--hicache-storage-backend-extra-config`, `tenant_id` in the JSON config file, or `MOONCAKE_TENANT_ID`.
> **Note:** strict isolation between tenants requires a Mooncake master started with `--enable_multi_tenants=true` and a tenant quota policy that explicitly registers each tenant. When strict multi-tenant mode is disabled, Mooncake ignores request tenant IDs for object placement and all objects use the `default` namespace. Non-default `tenant_id` also requires a Mooncake version that supports the `tenant_id` parameter in `MooncakeDistributedStore.setup()`. In `standalone_storage` mode, start the external `mooncake_client` with the matching `--tenant_id` because that process owns the real Mooncake client.
**SSD Offload (`enable_ssd_offload`):**
When `enable_ssd_offload` is set to `true`, SGLang will request that Mooncake enable SSD offloading for the KV cache. This allows Mooncake to spill overflow data from DRAM to local SSDs, effectively expanding the available L3 cache capacity.
python/sglang/srt/mem_cache/storage/mooncake_store/mooncake_embedding_store.py
+26 -10
View File
@@ -1,7 +1,10 @@
import logging
from typing import Any, List
from sglang.srt.mem_cache.storage.mooncake_store.mooncake_store import MooncakeBaseStore
from sglang.srt.mem_cache.storage.mooncake_store.mooncake_store import (
DEFAULT_TENANT_ID,
MooncakeBaseStore,
)
logger = logging.getLogger(__name__)
@@ -16,15 +19,28 @@ class MooncakeEmbeddingStore(MooncakeBaseStore):
MooncakeDistributedStore = self._import_mooncake_store()
self.store = MooncakeDistributedStore()
self.config = self._load_config(storage_config)
ret_code = self.store.setup(
self.config.local_hostname,
self.config.metadata_server,
self.config.global_segment_size,
16 * 1024 * 1024, # Internal local buffer size
self.config.protocol,
self.config.device_name,
self.config.master_server_address,
)
setup_kwargs = {}
if self.config.tenant_id != DEFAULT_TENANT_ID:
setup_kwargs["tenant_id"] = self.config.tenant_id
try:
ret_code = self.store.setup(
self.config.local_hostname,
self.config.metadata_server,
self.config.global_segment_size,
16 * 1024 * 1024, # Internal local buffer size
self.config.protocol,
self.config.device_name,
self.config.master_server_address,
**setup_kwargs,
)
except TypeError as e:
if "tenant_id" in setup_kwargs and "tenant_id" in str(e):
raise RuntimeError(
"The installed Mooncake version does not support tenant_id "
"in MooncakeDistributedStore.setup(). Please upgrade "
"Mooncake to use non-default Mooncake tenants with SGLang."
) from e
raise
if ret_code != 0:
raise RuntimeError(f"Failed to setup Mooncake Embedding Store: {ret_code}")
python/sglang/srt/mem_cache/storage/mooncake_store/mooncake_store.py
+25
View File
@@ -27,6 +27,7 @@ from sglang.srt.observability.metrics_collector import StorageMetrics
DEFAULT_LOCAL_BUFFER_SIZE = 16 * 1024 * 1024 # 16 MB
SETUP_TIMEOUT = 600 # 10min
DEFAULT_TENANT_ID = "default"
logger = logging.getLogger(__name__)
@@ -82,6 +83,13 @@ def _parse_global_segment_size(value) -> int:
return int(value)
def _normalize_tenant_id(value) -> str:
if value is None:
return DEFAULT_TENANT_ID
tenant_id = str(value).strip()
return tenant_id if tenant_id else DEFAULT_TENANT_ID
@dataclass
class MooncakeStoreConfig:
local_hostname: str
@@ -96,6 +104,7 @@ class MooncakeStoreConfig:
client_server_address: str
enable_ssd_offload: bool = False
ssd_offload_path: Optional[str] = None
tenant_id: str = DEFAULT_TENANT_ID
@staticmethod
def from_file() -> "MooncakeStoreConfig":
@@ -152,6 +161,9 @@ class MooncakeStoreConfig:
ssd_offload_path=config.get(
"ssd_offload_path", envs.MOONCAKE_OFFLOAD_FILE_STORAGE_PATH.default
),
tenant_id=_normalize_tenant_id(
config.get("tenant_id", envs.MOONCAKE_TENANT_ID.default)
),
)
@staticmethod
@@ -193,6 +205,7 @@ class MooncakeStoreConfig:
client_server_address=envs.MOONCAKE_CLIENT.get(),
enable_ssd_offload=envs.MOONCAKE_ENABLE_SSD_OFFLOAD.get(),
ssd_offload_path=envs.MOONCAKE_OFFLOAD_FILE_STORAGE_PATH.get(),
tenant_id=_normalize_tenant_id(envs.MOONCAKE_TENANT_ID.get()),
)
@staticmethod
@@ -241,6 +254,9 @@ class MooncakeStoreConfig:
ssd_offload_path=extra_config.get(
"ssd_offload_path", envs.MOONCAKE_OFFLOAD_FILE_STORAGE_PATH.default
),
tenant_id=_normalize_tenant_id(
extra_config.get("tenant_id", envs.MOONCAKE_TENANT_ID.default)
),
)
@@ -486,6 +502,8 @@ class MooncakeStore(HiCacheStorage, MooncakeBaseStore):
setup_kwargs["enable_ssd_offload"] = True
if self.config.ssd_offload_path is not None:
setup_kwargs["ssd_offload_path"] = self.config.ssd_offload_path
if self.config.tenant_id != DEFAULT_TENANT_ID:
setup_kwargs["tenant_id"] = self.config.tenant_id
while True:
try:
@@ -507,6 +525,13 @@ class MooncakeStore(HiCacheStorage, MooncakeBaseStore):
]
if not unsupported_kwargs:
raise
if "tenant_id" in unsupported_kwargs:
raise RuntimeError(
"The installed Mooncake version does not support "
"tenant_id in MooncakeDistributedStore.setup(). "
"Please upgrade Mooncake to use non-default "
"Mooncake tenants with SGLang."
) from e
logger.warning(
"The installed Mooncake version does not support the "
f"{', '.join(unsupported_kwargs)} parameter(s) in setup(). "