[HiCache] Add SeaweedFS L3 storage backend (#42399)

Co-authored-by: Zhiqiang Xie <xiezhq@stanford.edu>
This commit is contained in:
Ping Qiu
2026-10-07 15:05:37 -07:00
committed by GitHub
co-authored by Zhiqiang Xie
parent 0b635266d4
commit 685d4d453d
7 changed files with 936 additions and 1 deletions
@@ -145,7 +145,7 @@ class Memory(msgspec.Struct):
hicache_storage_backend: A[
Optional[str],
Arg(
help="The storage backend for hierarchical KV cache. Built-in backends: file, mooncake, npu_memcache, hf3fs, nixl, aibrix, tensorcast. For dynamic backend, use --hicache-storage-backend-extra-config to specify: backend_name (custom name), module_path (Python module path), class_name (backend class name).",
help="The storage backend for hierarchical KV cache. Built-in backends: file, mooncake, npu_memcache, hf3fs, nixl, aibrix, tensorcast, seaweedfs. For dynamic backend, use --hicache-storage-backend-extra-config to specify: backend_name (custom name), module_path (Python module path), class_name (backend class name).",
choices=[
"file",
"sim",
@@ -160,6 +160,7 @@ class Memory(msgspec.Struct):
"mori",
"shm",
"tensorcast",
"seaweedfs",
],
),
] = None
@@ -614,6 +614,7 @@ class HiCacheController:
"simm",
"mori",
"tensorcast",
"seaweedfs",
]
) or (
self.storage_backend_type == "dynamic"
@@ -192,6 +192,8 @@ class StorageBackendFactory:
return backend_class(storage_config, mem_pool_host)
elif backend_name == "shm":
return backend_class(storage_config, mem_pool_host)
elif backend_name == "seaweedfs":
return backend_class(storage_config, mem_pool_host)
elif backend_name == "tensorcast":
return backend_class(storage_config)
else:
@@ -266,3 +268,9 @@ StorageBackendFactory.register_backend(
"sglang.srt.mem_cache.storage.tensorcast_store.tensorcast_store",
"TensorcastStore",
)
StorageBackendFactory.register_backend(
"seaweedfs",
"sglang.srt.mem_cache.storage.seaweedfs.seaweedfs_store",
"SeaweedFSStore",
)
+89
View File
@@ -0,0 +1,89 @@
# SeaweedFS as the HiCache L3 backend
This backend stores HiCache KV pages in [SeaweedFS](https://github.com/seaweedfs/seaweedfs)
through its S3 gateway. Each KV page is one object. Every SGLang instance pointed at the same
bucket shares the same L3 cache, so a prefix computed by one instance can be prefetched by
another instance, including one on a different node.
## Requirements
- A SeaweedFS cluster with the S3 gateway enabled. For a quick single-node trial:
```bash
weed server -dir=/data/seaweedfs -s3 -s3.port=8333
```
- `boto3` in the SGLang environment:
```bash
pip install boto3
```
## Launch
```bash
python -m sglang.launch_server \
--model-path Qwen/Qwen3-8B \
--enable-hierarchical-cache \
--hicache-storage-backend seaweedfs \
--hicache-storage-backend-extra-config '{"endpoint": "http://seaweedfs-s3:8333", "bucket": "sglang-hicache"}'
```
`--hicache-storage-backend-extra-config` also accepts `@path/to/config.json` (or YAML/TOML).
| Key | Default | Meaning |
|---|---|---|
| `endpoint` | required | URL of the SeaweedFS S3 gateway |
| `bucket` | `sglang-hicache` | Bucket for KV pages; created if it does not exist |
| `prefix` | empty | Extra key prefix, for example to separate deployments in one bucket |
| `region` | `us-east-1` | Region sent with S3 requests |
| `access_key`, `secret_key` | unset | S3 credentials; when unset, the standard AWS credential chain is used (`AWS_ACCESS_KEY_ID`/`AWS_SECRET_ACCESS_KEY`, shared credentials file, and so on) |
| `max_workers` | `16` | Concurrent S3 requests per rank |
Prefer the AWS credential chain over putting secrets in the command line.
## Tuning
- **Page size.** Each KV page is one object, so `--page-size` sets the object size and the number
of S3 requests per prefix. Small pages turn a long prefix into many small GETs, where per-request
overhead dominates. Start with `--page-size 64` or larger.
- **Prefetch threshold.** A prefetch from SeaweedFS pays off only when loading the prefix is faster
than recomputing it, which depends on the model, the GPU, the prefix length and the bandwidth to
SeaweedFS. The default `prefetch_threshold` of 256 tokens is low for a remote store; measure
time-to-first-token with and without the cache for a few prefix lengths and set the threshold at
the break-even point, for example:
```bash
--hicache-storage-backend-extra-config '{"endpoint": "http://seaweedfs-s3:8333", "prefetch_threshold": 8192}'
```
`prefetch_threshold`, `prefetch_timeout_base` and `prefetch_timeout_per_ki_token` are read by
SGLang's cache controller, not by this backend.
## Object layout
Objects are named `<prefix>/<model>/<rank scope>/<page hash>`:
- MHA models include `tp<rank>of<size>`, because tensor-parallel ranks hold different heads.
- MLA models omit the tp rank, because every rank holds identical KV and only one rank backs up.
- Pipeline-parallel (`pp<rank>of<size>`) and context-parallel (`cp<rank>of<size>`) components are
added when those degrees are greater than one.
Instances share cache entries only when the model name and parallel layout match.
Hybrid models (Mamba, SWA, DeepSeek V4 and other side pools) store each side-pool page next to
its KV page as `<page hash>.<pool>`. KV pages keep the same names on both interfaces, so one
cache serves both.
## Behaviour and limits
- The backend uses the zero-copy HiCache interface: pages move directly between SeaweedFS and the
host KV pool's own buffers, without a staging page.
- Hybrid models use the v2 interface (`batch_exists_v2`, `batch_get_v2`, `batch_set_v2`). A side
pool with the `trailing_pages` policy only needs the pages in its window to be present.
- A prefetch stops at the first missing page. Pages after a miss are not loaded.
- An object whose size does not match the host page is treated as a miss and never copied.
- `clear()` deletes only the calling rank's objects. Other models and ranks sharing the bucket are
left alone.
- The backend does not evict. Size the SeaweedFS cluster for the cache, or apply a retention
policy on the SeaweedFS side.
@@ -0,0 +1,431 @@
"""SeaweedFS backend for HiCache L3 storage.
Each KV page is one object in a SeaweedFS bucket, reached through SeaweedFS's S3 gateway. The
cache controller drives the zero-copy interface: ``batch_get_v1`` / ``batch_set_v1`` move a
page between its object and the host pool's own buffers (from ``get_page_buffer_meta``),
returning one success flag per page. An object holds the page's buffers back to back, which
for the page-first layouts is byte-identical to the flat page of the generic ``batch_get`` /
``batch_set`` interface, so the two interfaces read each other's objects.
``batch_exists`` returns the length of the leading run of existing keys.
Configuration comes from ``--hicache-storage-backend-extra-config``; only ``endpoint`` is
required. When ``access_key``/``secret_key`` are absent the standard AWS credential chain is
used, which keeps secrets off the command line.
"""
from __future__ import annotations
import ctypes
import hashlib
import logging
from concurrent.futures import ThreadPoolExecutor
from typing import Any, Dict, List, Optional
import msgspec
import torch
from sglang.srt.mem_cache.hicache_storage import (
HiCacheStorage,
HiCacheStorageConfig,
HiCacheStorageExtraInfo,
PoolHitPolicy,
PoolName,
PoolTransfer,
PoolTransferResult,
)
logger = logging.getLogger(__name__)
# S3 keys are capped at 1024 bytes; longer page keys are hashed down below this.
_MAX_KEY_LEN = 900
_MISSING_CODES = ("404", "NoSuchKey", "NoSuchBucket", "NotFound")
class SeaweedFSConfig(msgspec.Struct, frozen=True, kw_only=True):
endpoint: str
bucket: str = "sglang-hicache"
prefix: str = ""
region: str = "us-east-1"
access_key: Optional[str] = None
secret_key: Optional[str] = None
max_workers: int = 16
@classmethod
def from_extra_config(cls, extra_config: Optional[dict]) -> SeaweedFSConfig:
# extra_config also carries factory keys (backend_name, module_path, ...).
fields = set(cls.__struct_fields__)
return msgspec.convert(
{k: v for k, v in (extra_config or {}).items() if k in fields}, type=cls
)
def _make_s3_client(config: SeaweedFSConfig):
import boto3
from botocore.config import Config as BotoConfig
credentials = {}
if config.access_key and config.secret_key:
credentials = {
"aws_access_key_id": config.access_key,
"aws_secret_access_key": config.secret_key,
}
return boto3.client(
"s3",
endpoint_url=config.endpoint,
region_name=config.region,
# Path-style: virtual-hosted addressing needs per-bucket DNS a cluster gateway lacks.
config=BotoConfig(
s3={"addressing_style": "path"},
max_pool_connections=max(config.max_workers, 10),
retries={"max_attempts": 3, "mode": "standard"},
),
**credentials,
)
def key_scope(storage_config: HiCacheStorageConfig) -> List[str]:
"""Model and rank components of an object key; mirrors HiCacheFile's suffix rules."""
parts = []
if storage_config.model_name:
parts.append("-".join(storage_config.model_name.split("/")))
# MLA ranks hold identical KV and only one of them backs up, so they share keys;
# MHA ranks hold different heads and must not.
if not storage_config.is_mla_model:
parts.append(f"tp{storage_config.tp_rank}of{storage_config.tp_size}")
if storage_config.pp_size > 1:
parts.append(f"pp{storage_config.pp_rank}of{storage_config.pp_size}")
if storage_config.attn_cp_size > 1:
parts.append(f"cp{storage_config.attn_cp_rank}of{storage_config.attn_cp_size}")
return parts
def _byte_view(tensor: torch.Tensor) -> memoryview:
# Through uint8 so bf16/fp8 pages work; numpy has no such dtypes.
return memoryview(tensor.reshape(-1).view(torch.uint8).numpy())
class SeaweedFSStore(HiCacheStorage):
def __init__(
self, storage_config: HiCacheStorageConfig, mem_pool_host: Any = None
) -> None:
self.config = SeaweedFSConfig.from_extra_config(storage_config.extra_config)
scope = [self.config.prefix.strip("/")] if self.config.prefix else []
self.key_prefix = "/".join(scope + key_scope(storage_config))
self._s3 = _make_s3_client(self.config)
self._executor = ThreadPoolExecutor(max_workers=self.config.max_workers)
self._ensure_bucket()
logger.info(
"SeaweedFS HiCache backend: endpoint=%s bucket=%s prefix=%s",
self.config.endpoint,
self.config.bucket,
self.key_prefix,
)
def _is_missing(self, exc: Exception) -> bool:
return (
isinstance(exc, self._s3.exceptions.ClientError)
and str(exc.response["Error"]["Code"]) in _MISSING_CODES
)
def _ensure_bucket(self) -> None:
try:
self._s3.head_bucket(Bucket=self.config.bucket)
except Exception as exc:
if not self._is_missing(exc):
raise
self._s3.create_bucket(Bucket=self.config.bucket)
def _object_key(self, key: str) -> str:
full = f"{self.key_prefix}/{key}" if self.key_prefix else key
if len(full) <= _MAX_KEY_LEN:
return full
return f"{full[:800]}/{hashlib.sha256(full.encode()).hexdigest()}"
def _fetch(self, key: str) -> Optional[bytes]:
try:
resp = self._s3.get_object(
Bucket=self.config.bucket, Key=self._object_key(key)
)
return resp["Body"].read()
except Exception as exc:
if not self._is_missing(exc):
logger.warning("SeaweedFS get %s failed: %s", key, exc)
return None
def get(
self,
key: str,
target_location: Optional[torch.Tensor] = None,
target_sizes: Optional[Any] = None,
) -> torch.Tensor | None:
data = self._fetch(key)
if data is None:
return None
if target_location is None:
return torch.frombuffer(bytearray(data), dtype=torch.uint8)
nbytes = target_location.numel() * target_location.element_size()
# A size mismatch is a stale or foreign object; copying it would overrun the page.
if len(data) != nbytes or not target_location.is_contiguous():
logger.warning(
"SeaweedFS object %s is %d bytes, page is %d", key, len(data), nbytes
)
return None
_byte_view(target_location)[:] = data
return target_location
def batch_get(
self,
keys: List[str],
target_locations: Optional[List[torch.Tensor]] = None,
target_sizes: Optional[Any] = None,
) -> List[torch.Tensor | None]:
locations = target_locations or [None] * len(keys)
futures = [
self._executor.submit(self.get, key, location)
for key, location in zip(keys, locations)
]
return [f.result() for f in futures]
def set(
self,
key: str,
value: Optional[torch.Tensor] = None,
target_location: Optional[torch.Tensor] = None,
target_sizes: Optional[Any] = None,
) -> bool:
page = value if value is not None else target_location
try:
body = bytes(_byte_view(page.contiguous()))
self._s3.put_object(
Bucket=self.config.bucket, Key=self._object_key(key), Body=body
)
return True
except Exception as exc:
logger.error("SeaweedFS set %s failed: %s", key, exc)
return False
def batch_set(
self,
keys: List[str],
values: Optional[List[torch.Tensor]] = None,
target_locations: Optional[List[torch.Tensor]] = None,
target_sizes: Optional[Any] = None,
) -> bool:
pages = values if values is not None else target_locations
futures = [
self._executor.submit(self.set, key, page) for key, page in zip(keys, pages)
]
return all([f.result() for f in futures])
def _page_buffers(self, host_indices: torch.Tensor) -> List[List[tuple]]:
"""(address, nbytes) of each host-pool buffer of every page, in page order."""
addresses, sizes = self.mem_pool_host.get_page_buffer_meta(host_indices)
num_pages = len(host_indices) // self.mem_pool_host.page_size
per_page = len(addresses) // num_pages
if isinstance(sizes, int):
sizes = [sizes] * len(addresses)
return [
list(
zip(
addresses[i * per_page : (i + 1) * per_page],
sizes[i * per_page : (i + 1) * per_page],
)
)
for i in range(num_pages)
]
def _get_into(self, key: str, buffers: List[tuple]) -> bool:
data = self._fetch(key)
if data is None:
return False
expected = sum(nbytes for _, nbytes in buffers)
# A size mismatch is a stale or foreign object; copying it would overrun the page.
if len(data) != expected:
logger.warning(
"SeaweedFS object %s is %d bytes, page is %d", key, len(data), expected
)
return False
src = ctypes.cast(ctypes.c_char_p(data), ctypes.c_void_p).value
offset = 0
for address, nbytes in buffers:
ctypes.memmove(address, src + offset, nbytes)
offset += nbytes
return True
def _put_from(self, key: str, buffers: List[tuple]) -> bool:
try:
body = bytearray(sum(nbytes for _, nbytes in buffers))
dst = ctypes.addressof(ctypes.c_char.from_buffer(body))
offset = 0
for address, nbytes in buffers:
ctypes.memmove(dst + offset, address, nbytes)
offset += nbytes
self._s3.put_object(
Bucket=self.config.bucket, Key=self._object_key(key), Body=body
)
return True
except Exception as exc:
logger.error("SeaweedFS set %s failed: %s", key, exc)
return False
def batch_get_v1(
self,
keys: List[str],
host_indices: torch.Tensor,
extra_info: Optional[HiCacheStorageExtraInfo] = None,
) -> List[bool]:
pages = self._page_buffers(host_indices)
futures = [
self._executor.submit(self._get_into, key, buffers)
for key, buffers in zip(keys, pages)
]
return [f.result() for f in futures]
def batch_set_v1(
self,
keys: List[str],
host_indices: torch.Tensor,
extra_info: Optional[HiCacheStorageExtraInfo] = None,
) -> List[bool]:
pages = self._page_buffers(host_indices)
futures = [
self._executor.submit(self._put_from, key, buffers)
for key, buffers in zip(keys, pages)
]
return [f.result() for f in futures]
# Hybrid models (Mamba, SWA, DeepSeek V4 indexer and other side pools) go
# through the v2 interface. Side-pool pages are stored next to their KV page
# as "<key>.<pool>"; KV pages keep the same object names as v1, so a cache
# written by either path is readable by the other.
def _component_key(self, key: str, pool_name) -> str:
name = getattr(pool_name, "value", pool_name)
return key if name == PoolName.KV.value else f"{key}.{name}"
def batch_exists_v2(
self,
keys: List[str],
pool_transfers: Optional[List[PoolTransfer]] = None,
extra_info: Optional[HiCacheStorageExtraInfo] = None,
) -> PoolTransferResult:
kv_pages = self.batch_exists(keys, extra_info)
hit_count: Dict[str, int] = {PoolName.KV: kv_pages} if kv_pages else {}
final_pages = kv_pages
for transfer in pool_transfers or []:
if final_pages == 0:
break
futures = [
self._executor.submit(
self.exists, self._component_key(keys[i], transfer.name)
)
for i in range(kv_pages)
]
present = [f.result() for f in futures]
if transfer.hit_policy == PoolHitPolicy.ALL_PAGES:
boundary = next((i for i, ok in enumerate(present) if not ok), kv_pages)
else: # TRAILING_PAGES: only the window ending at the prefix must exist
trailing = max(1, len(transfer.keys) if transfer.keys else 1)
boundary = next(
(
n
for n in range(kv_pages, 0, -1)
if all(present[max(0, n - trailing) : n])
),
0,
)
if boundary:
hit_count[transfer.name] = boundary
final_pages = min(final_pages, boundary)
return PoolTransferResult(final_pages, hit_count)
def _read_page_v2(self, pool_name, key: str, host_pool, offset: int) -> bool:
page = host_pool.get_dummy_flat_data_page()
if self.get(self._component_key(key, pool_name), page) is None:
return False
host_pool.set_from_flat_data_page(offset, page)
return True
def _write_page_v2(self, pool_name, key: str, host_pool, offset: int) -> bool:
page = host_pool.get_data_page(offset, flat=True)
return self.set(self._component_key(key, pool_name), page)
def _batch_io_v2(self, transfers: List[PoolTransfer], page_fn):
results: Dict[str, List[bool]] = {}
pending = []
for transfer in transfers:
keys = transfer.keys or []
host_pool = getattr(self, "registered_pools", {}).get(transfer.name)
if host_pool is None:
logger.error("SeaweedFS: host pool %s is not registered", transfer.name)
results[transfer.name] = [False] * len(keys)
continue
page_size = getattr(host_pool, "page_size", 1) or 1
indices = transfer.host_indices
if indices is None or indices.numel() != len(keys) * page_size:
logger.error(
"SeaweedFS: %s has %d keys but %s host indices (page size %d)",
transfer.name,
len(keys),
indices.numel() if indices is not None else 0,
page_size,
)
results[transfer.name] = [False] * len(keys)
continue
futures = [
self._executor.submit(
page_fn,
transfer.name,
key,
host_pool,
indices[i * page_size].item(),
)
for i, key in enumerate(keys)
]
pending.append((transfer.name, futures))
for name, futures in pending:
results[name] = [f.result() for f in futures]
return results
def batch_get_v2(
self,
transfers: List[PoolTransfer],
extra_info: Optional[HiCacheStorageExtraInfo] = None,
) -> Dict[str, List[bool]]:
return self._batch_io_v2(transfers, self._read_page_v2)
def batch_set_v2(
self,
transfers: List[PoolTransfer],
extra_info: Optional[HiCacheStorageExtraInfo] = None,
) -> Dict[str, List[bool]]:
return self._batch_io_v2(transfers, self._write_page_v2)
def exists(self, key: str) -> bool:
try:
self._s3.head_object(Bucket=self.config.bucket, Key=self._object_key(key))
return True
except Exception as exc:
if not self._is_missing(exc):
logger.warning("SeaweedFS exists %s failed: %s", key, exc)
return False
def batch_exists(
self, keys: List[str], extra_info: Optional[HiCacheStorageExtraInfo] = None
) -> int:
futures = [self._executor.submit(self.exists, key) for key in keys]
for i, f in enumerate(futures):
if not f.result():
return i
return len(keys)
def clear(self) -> None:
# Only this rank's prefix; other models and ranks may share the bucket.
prefix = f"{self.key_prefix}/" if self.key_prefix else ""
paginator = self._s3.get_paginator("list_objects_v2")
for page in paginator.paginate(Bucket=self.config.bucket, Prefix=prefix):
objects = [{"Key": o["Key"]} for o in page.get("Contents", [])]
if objects:
self._s3.delete_objects(
Bucket=self.config.bucket, Delete={"Objects": objects}
)
@@ -0,0 +1,405 @@
"""Unit tests for the SeaweedFS HiCache storage backend.
Pages move through the cache controller's own generic get/set paths and a real CPU host pool;
only the S3 client is replaced, by an in-memory store with S3's miss semantics.
Run with:
python3 -m pytest test/registered/unit/mem_cache/test_hicache_seaweedfs_storage.py -v
"""
from sglang.test.ci.ci_register import register_cpu_ci
register_cpu_ci(est_time=10, suite="base-a-test-cpu")
import io
import unittest
import uuid
from types import SimpleNamespace
from unittest import mock
import torch
from sglang.srt.managers.cache_controller import HiCacheController
from sglang.srt.mem_cache.hicache_storage import (
HiCacheStorageConfig,
HiCacheStorageExtraInfo,
PoolHitPolicy,
PoolName,
PoolTransfer,
)
from sglang.srt.mem_cache.pool_host.mha import MHATokenToKVPoolHost
from sglang.srt.mem_cache.storage import backend_factory
from sglang.srt.mem_cache.storage.seaweedfs import seaweedfs_store
from sglang.test.test_utils import CustomTestCase
PAGE_SIZE = 16
NUM_PAGES = 4
class _ClientError(Exception):
def __init__(self, code: str):
super().__init__(code)
self.response = {"Error": {"Code": code}}
class InMemoryS3:
"""The subset of the boto3 S3 client the backend uses, with S3's miss behaviour."""
exceptions = SimpleNamespace(ClientError=_ClientError)
def __init__(self, buckets: dict):
self.buckets = buckets
def _bucket(self, bucket):
if bucket not in self.buckets:
raise _ClientError("NoSuchBucket")
return self.buckets[bucket]
def head_bucket(self, Bucket):
self._bucket(Bucket)
def create_bucket(self, Bucket):
self.buckets.setdefault(Bucket, {})
def put_object(self, Bucket, Key, Body):
self._bucket(Bucket)[Key] = bytes(Body)
def get_object(self, Bucket, Key):
objects = self._bucket(Bucket)
if Key not in objects:
raise _ClientError("NoSuchKey")
return {"Body": io.BytesIO(objects[Key])}
def head_object(self, Bucket, Key):
if Key not in self._bucket(Bucket):
raise _ClientError("404")
def delete_object(self, Bucket, Key):
self._bucket(Bucket).pop(Key, None)
def delete_objects(self, Bucket, Delete):
for obj in Delete["Objects"]:
self._bucket(Bucket).pop(obj["Key"], None)
def get_paginator(self, name):
store = self
class _Paginator:
def paginate(self, Bucket, Prefix):
keys = [k for k in store._bucket(Bucket) if k.startswith(Prefix)]
yield {"Contents": [{"Key": k} for k in keys]}
return _Paginator()
def _host_pool(layout: str) -> MHATokenToKVPoolHost:
device_pool = SimpleNamespace(
store_dtype=torch.bfloat16,
size=64,
layer_num=4,
start_layer=0,
end_layer=4,
head_num=2,
head_dim=64,
v_head_dim=64,
row_dim=128,
device="cpu",
layer_shard_enabled=False,
hicache_write_back_staging=None,
)
return MHATokenToKVPoolHost(
device_pool=device_pool,
host_to_device_ratio=2.0,
host_size=0,
page_size=PAGE_SIZE,
layout=layout,
pin_memory=False,
device="cpu",
)
def _page(pool: MHATokenToKVPoolHost, page: int) -> torch.Tensor:
return pool.get_data_page(page * PAGE_SIZE, flat=True).view(torch.uint8).clone()
class TestSeaweedFSStore(CustomTestCase):
endpoint = "http://seaweedfs.invalid:8333"
def make_client(self, config):
return InMemoryS3(self.buckets)
def setUp(self):
self.buckets = {}
self.bucket = f"sgl-{uuid.uuid4().hex[:10]}"
patcher = mock.patch.object(
seaweedfs_store,
"_make_s3_client",
side_effect=lambda config: self.make_client(config),
)
patcher.start()
self.addCleanup(patcher.stop)
def _config(self, tp_rank=0, tp_size=1, is_mla=False, cp_rank=0, cp_size=1):
return HiCacheStorageConfig(
tp_rank=tp_rank,
tp_size=tp_size,
pp_rank=0,
pp_size=1,
attn_cp_rank=cp_rank,
attn_cp_size=cp_size,
is_mla_model=is_mla,
enable_storage_metrics=False,
is_page_first_layout=True,
model_name="Qwen/Qwen3-0.6B",
extra_config={"endpoint": self.endpoint, "bucket": self.bucket},
)
def _store(self, **kwargs):
return backend_factory.StorageBackendFactory.create_backend(
backend_name="seaweedfs",
storage_config=self._config(**kwargs),
mem_pool_host=None,
)
def _controller(self, store, pool):
store.register_mem_pool_host(pool)
return SimpleNamespace(
storage_backend=store, storage_host_pool=pool, page_size=PAGE_SIZE
)
def _backup_and_reload(
self, store, layout, drop_page=None, set_via="generic", get_via="generic"
):
"""Back up NUM_PAGES random pages, wipe the pool, prefetch them back.
set_via / get_via pick the controller path: "generic" (flat page tensors) or
"zero_copy" (batch_get_v1 / batch_set_v1 on the host pool's own buffers).
"""
pool = _host_pool(layout)
ctrl = self._controller(store, pool)
pool.kv_buffer.copy_(torch.randn(pool.kv_buffer.shape).to(torch.bfloat16))
expected = [_page(pool, i) for i in range(NUM_PAGES)]
keys = [f"{layout}-{uuid.uuid4().hex[:8]}-{i}" for i in range(NUM_PAGES)]
host_indices = torch.arange(NUM_PAGES * PAGE_SIZE, dtype=torch.int64)
page_set = {
"generic": HiCacheController._generic_page_set,
"zero_copy": HiCacheController._page_set_zero_copy,
}[set_via]
page_get = {
"generic": HiCacheController._generic_page_get,
"zero_copy": HiCacheController._page_get_zero_copy,
}[get_via]
self.assertTrue(page_set(ctrl, keys, host_indices))
if drop_page is not None:
store._s3.delete_object(
Bucket=self.bucket, Key=store._object_key(keys[drop_page])
)
pool.kv_buffer.zero_()
operation = SimpleNamespace(request_id="r", is_terminated=lambda: False)
loaded = page_get(ctrl, operation, keys, host_indices)
return pool, keys, expected, loaded
def test_controller_round_trip_is_bit_exact_for_every_layout(self):
store = self._store()
for via in ("generic", "zero_copy"):
for layout in ("layer_first", "page_first", "page_first_direct"):
with self.subTest(via=via, layout=layout):
pool, _, expected, loaded = self._backup_and_reload(
store, layout, set_via=via, get_via=via
)
self.assertEqual(loaded, NUM_PAGES)
for i in range(NUM_PAGES):
self.assertTrue(torch.equal(_page(pool, i), expected[i]))
def test_generic_and_zero_copy_read_each_others_pages(self):
"""Both interfaces store a page as one object with the same bytes (page-first layouts)."""
store = self._store()
for layout in ("page_first", "page_first_direct"):
for set_via, get_via in (
("generic", "zero_copy"),
("zero_copy", "generic"),
):
with self.subTest(layout=layout, set_via=set_via):
pool, _, expected, loaded = self._backup_and_reload(
store, layout, set_via=set_via, get_via=get_via
)
self.assertEqual(loaded, NUM_PAGES)
for i in range(NUM_PAGES):
self.assertTrue(torch.equal(_page(pool, i), expected[i]))
def test_prefetch_stops_at_the_first_missing_page(self):
"""Pages after a miss are unusable and must not be loaded or counted."""
store = self._store()
for via in ("generic", "zero_copy"):
with self.subTest(via=via):
pool, keys, _, loaded = self._backup_and_reload(
store, "page_first", drop_page=2, set_via=via, get_via=via
)
self.assertEqual(loaded, 2)
if via == "generic":
# Zero-copy fetches a batch in parallel into slots the controller
# releases after the count; only the generic path skips the copy.
self.assertFalse(_page(pool, 3).any())
extra_info = HiCacheStorageExtraInfo(prefix_keys=[])
self.assertEqual(store.batch_exists(keys, extra_info), 2)
def test_zero_copy_rejects_an_object_of_the_wrong_size(self):
"""A stale or foreign object must not be copied into the host pool."""
store = self._store()
pool = _host_pool("page_first")
pool.kv_buffer.zero_()
store.register_mem_pool_host(pool)
store._s3.put_object(
Bucket=self.bucket, Key=store._object_key("k"), Body=b"\x7f" * 8192
)
host_indices = torch.arange(PAGE_SIZE, dtype=torch.int64)
self.assertEqual(store.batch_get_v1(["k"], host_indices), [False])
self.assertFalse(pool.kv_buffer.any())
def test_object_of_the_wrong_size_is_a_miss_not_an_overrun(self):
store = self._store()
store._s3.put_object(
Bucket=self.bucket, Key=store._object_key("k"), Body=b"\x7f" * 8192
)
arena = torch.zeros(8192, dtype=torch.uint8)
self.assertIsNone(store.get("k", arena[:4096]))
self.assertFalse(arena.any())
def test_mha_and_context_parallel_ranks_do_not_share_objects(self):
for kwargs in (
[dict(tp_rank=0, tp_size=2), dict(tp_rank=1, tp_size=2)],
[dict(cp_rank=0, cp_size=2), dict(cp_rank=1, cp_size=2)],
):
with self.subTest(ranks=kwargs):
first, second = self._store(**kwargs[0]), self._store(**kwargs[1])
first.set("shared", torch.full((64,), 1, dtype=torch.uint8))
second.set("shared", torch.full((64,), 2, dtype=torch.uint8))
self.assertEqual(int(first.get("shared")[0]), 1)
second.clear()
self.assertTrue(first.exists("shared"))
def test_mla_ranks_share_objects(self):
"""MLA ranks hold identical KV and only one backs up, so the others must find it."""
writer = self._store(tp_rank=0, tp_size=2, is_mla=True)
reader = self._store(tp_rank=1, tp_size=2, is_mla=True)
writer.set("shared", torch.full((64,), 3, dtype=torch.uint8))
self.assertTrue(reader.exists("shared"))
# ---- v2 interface (hybrid models): KV plus a side pool -----------------
# A second real host pool stands in for the side pool (Mamba/SWA/indexer):
# the backend only touches it through the host-pool page API, so what this
# proves is the per-pool object naming, routing and hit policies.
def _v2_store(self, layout="page_first"):
store = self._store()
kv, side = _host_pool(layout), _host_pool(layout)
store.register_mem_host_pool_v2(kv, PoolName.KV)
store.register_mem_host_pool_v2(side, PoolName.MAMBA)
for pool in (kv, side):
pool.kv_buffer.copy_(torch.randn(pool.kv_buffer.shape).to(torch.bfloat16))
keys = [f"v2-{uuid.uuid4().hex[:8]}-{i}" for i in range(NUM_PAGES)]
indices = torch.arange(NUM_PAGES * PAGE_SIZE, dtype=torch.int64)
transfers = [
PoolTransfer(name=PoolName.KV, host_indices=indices, keys=keys),
PoolTransfer(name=PoolName.MAMBA, host_indices=indices, keys=keys),
]
return store, kv, side, keys, transfers
def test_v2_round_trip_is_bit_exact_per_pool(self):
for layout in ("layer_first", "page_first", "page_first_direct"):
with self.subTest(layout=layout):
store, kv, side, keys, transfers = self._v2_store(layout)
expected = {
p: [_page(pool, i) for i in range(NUM_PAGES)]
for p, pool in ((PoolName.KV, kv), (PoolName.MAMBA, side))
}
written = store.batch_set_v2(transfers)
self.assertEqual(written[PoolName.KV], [True] * NUM_PAGES)
self.assertEqual(written[PoolName.MAMBA], [True] * NUM_PAGES)
kv.kv_buffer.zero_()
side.kv_buffer.zero_()
loaded = store.batch_get_v2(transfers)
for p, pool in ((PoolName.KV, kv), (PoolName.MAMBA, side)):
self.assertEqual(loaded[p], [True] * NUM_PAGES)
for i in range(NUM_PAGES):
self.assertTrue(torch.equal(_page(pool, i), expected[p][i]))
def test_v2_kv_pages_are_the_v1_objects(self):
# KV pages keep their v1 object names, so v1 and v2 share one cache;
# side-pool pages are separate objects and never overwrite them.
store, kv, side, keys, transfers = self._v2_store()
store.register_mem_pool_host(kv) # the v1 path reads through this pool
store.batch_set_v2(transfers)
self.assertEqual(store.batch_exists(keys), NUM_PAGES)
self.assertEqual(store.batch_exists([f"{k}.mamba" for k in keys]), NUM_PAGES)
self.assertEqual(
store.batch_get_v1(keys, transfers[0].host_indices), [True] * NUM_PAGES
)
def test_v2_exists_all_pages_stops_at_a_missing_side_page(self):
store, _, _, keys, transfers = self._v2_store()
store.batch_set_v2(transfers)
store._s3.delete_object(
Bucket=self.bucket, Key=store._object_key(f"{keys[2]}.mamba")
)
side = PoolTransfer(
name=PoolName.MAMBA, keys=keys, hit_policy=PoolHitPolicy.ALL_PAGES
)
result = store.batch_exists_v2(keys, [side])
self.assertEqual(result.kv_hit_pages, 2)
self.assertEqual(
result.extra_pool_hit_pages, {PoolName.KV: NUM_PAGES, PoolName.MAMBA: 2}
)
def test_v2_exists_trailing_pages_needs_only_the_window(self):
store, _, _, keys, transfers = self._v2_store()
store.batch_set_v2(transfers)
# Only the last page's side state matters for a one-page window, so a
# gap earlier in the side pool does not shorten the prefix...
store._s3.delete_object(
Bucket=self.bucket, Key=store._object_key(f"{keys[0]}.mamba")
)
window = PoolTransfer(
name=PoolName.MAMBA, keys=keys[-1:], hit_policy=PoolHitPolicy.TRAILING_PAGES
)
self.assertEqual(store.batch_exists_v2(keys, [window]).kv_hit_pages, NUM_PAGES)
# ...but losing the last page's state falls back to the longest prefix
# whose own last page still has it.
store._s3.delete_object(
Bucket=self.bucket, Key=store._object_key(f"{keys[-1]}.mamba")
)
result = store.batch_exists_v2(keys, [window])
self.assertEqual(result.kv_hit_pages, NUM_PAGES - 1)
self.assertEqual(result.extra_pool_hit_pages[PoolName.MAMBA], NUM_PAGES - 1)
def test_v2_exists_without_kv_prefix_is_empty(self):
store, _, _, keys, _ = self._v2_store()
side = PoolTransfer(name=PoolName.MAMBA, keys=keys)
result = store.batch_exists_v2(keys, [side])
self.assertEqual((result.kv_hit_pages, result.extra_pool_hit_pages), (0, {}))
def test_v2_rejects_mismatched_indices_and_unknown_pools(self):
store, _, _, keys, _ = self._v2_store()
short = torch.arange(PAGE_SIZE, dtype=torch.int64) # one page for four keys
bad = [
PoolTransfer(name=PoolName.KV, host_indices=short, keys=keys),
PoolTransfer(
name=PoolName.SWA,
host_indices=torch.arange(NUM_PAGES * PAGE_SIZE),
keys=keys,
),
]
for op in (store.batch_set_v2, store.batch_get_v2):
result = op(bad)
self.assertEqual(result[PoolName.KV], [False] * NUM_PAGES)
self.assertEqual(result[PoolName.SWA], [False] * NUM_PAGES)
self.assertEqual(store.batch_exists(keys), 0)
def test_missing_endpoint_is_refused(self):
with self.assertRaises(ValueError):
seaweedfs_store.SeaweedFSConfig.from_extra_config({"bucket": "b"})
if __name__ == "__main__":
unittest.main()