diff --git a/python/sglang/srt/arg_groups/fields/memory.py b/python/sglang/srt/arg_groups/fields/memory.py index 79d230d6be..c79a33346a 100644 --- a/python/sglang/srt/arg_groups/fields/memory.py +++ b/python/sglang/srt/arg_groups/fields/memory.py @@ -145,7 +145,7 @@ class Memory(msgspec.Struct): hicache_storage_backend: A[ Optional[str], Arg( - help="The storage backend for hierarchical KV cache. Built-in backends: file, mooncake, npu_memcache, hf3fs, nixl, aibrix, tensorcast. For dynamic backend, use --hicache-storage-backend-extra-config to specify: backend_name (custom name), module_path (Python module path), class_name (backend class name).", + help="The storage backend for hierarchical KV cache. Built-in backends: file, mooncake, npu_memcache, hf3fs, nixl, aibrix, tensorcast, seaweedfs. For dynamic backend, use --hicache-storage-backend-extra-config to specify: backend_name (custom name), module_path (Python module path), class_name (backend class name).", choices=[ "file", "sim", @@ -160,6 +160,7 @@ class Memory(msgspec.Struct): "mori", "shm", "tensorcast", + "seaweedfs", ], ), ] = None diff --git a/python/sglang/srt/managers/cache_controller.py b/python/sglang/srt/managers/cache_controller.py index 27b775d1f5..8a27dbf38b 100644 --- a/python/sglang/srt/managers/cache_controller.py +++ b/python/sglang/srt/managers/cache_controller.py @@ -614,6 +614,7 @@ class HiCacheController: "simm", "mori", "tensorcast", + "seaweedfs", ] ) or ( self.storage_backend_type == "dynamic" diff --git a/python/sglang/srt/mem_cache/storage/backend_factory.py b/python/sglang/srt/mem_cache/storage/backend_factory.py index c55ae39807..7537f28770 100644 --- a/python/sglang/srt/mem_cache/storage/backend_factory.py +++ b/python/sglang/srt/mem_cache/storage/backend_factory.py @@ -192,6 +192,8 @@ class StorageBackendFactory: return backend_class(storage_config, mem_pool_host) elif backend_name == "shm": return backend_class(storage_config, mem_pool_host) + elif backend_name == "seaweedfs": + return backend_class(storage_config, mem_pool_host) elif backend_name == "tensorcast": return backend_class(storage_config) else: @@ -266,3 +268,9 @@ StorageBackendFactory.register_backend( "sglang.srt.mem_cache.storage.tensorcast_store.tensorcast_store", "TensorcastStore", ) + +StorageBackendFactory.register_backend( + "seaweedfs", + "sglang.srt.mem_cache.storage.seaweedfs.seaweedfs_store", + "SeaweedFSStore", +) diff --git a/python/sglang/srt/mem_cache/storage/seaweedfs/README.md b/python/sglang/srt/mem_cache/storage/seaweedfs/README.md new file mode 100755 index 0000000000..46f0a98f49 --- /dev/null +++ b/python/sglang/srt/mem_cache/storage/seaweedfs/README.md @@ -0,0 +1,89 @@ +# SeaweedFS as the HiCache L3 backend + +This backend stores HiCache KV pages in [SeaweedFS](https://github.com/seaweedfs/seaweedfs) +through its S3 gateway. Each KV page is one object. Every SGLang instance pointed at the same +bucket shares the same L3 cache, so a prefix computed by one instance can be prefetched by +another instance, including one on a different node. + +## Requirements + +- A SeaweedFS cluster with the S3 gateway enabled. For a quick single-node trial: + + ```bash + weed server -dir=/data/seaweedfs -s3 -s3.port=8333 + ``` + +- `boto3` in the SGLang environment: + + ```bash + pip install boto3 + ``` + +## Launch + +```bash +python -m sglang.launch_server \ + --model-path Qwen/Qwen3-8B \ + --enable-hierarchical-cache \ + --hicache-storage-backend seaweedfs \ + --hicache-storage-backend-extra-config '{"endpoint": "http://seaweedfs-s3:8333", "bucket": "sglang-hicache"}' +``` + +`--hicache-storage-backend-extra-config` also accepts `@path/to/config.json` (or YAML/TOML). + +| Key | Default | Meaning | +|---|---|---| +| `endpoint` | required | URL of the SeaweedFS S3 gateway | +| `bucket` | `sglang-hicache` | Bucket for KV pages; created if it does not exist | +| `prefix` | empty | Extra key prefix, for example to separate deployments in one bucket | +| `region` | `us-east-1` | Region sent with S3 requests | +| `access_key`, `secret_key` | unset | S3 credentials; when unset, the standard AWS credential chain is used (`AWS_ACCESS_KEY_ID`/`AWS_SECRET_ACCESS_KEY`, shared credentials file, and so on) | +| `max_workers` | `16` | Concurrent S3 requests per rank | + +Prefer the AWS credential chain over putting secrets in the command line. + +## Tuning + +- **Page size.** Each KV page is one object, so `--page-size` sets the object size and the number + of S3 requests per prefix. Small pages turn a long prefix into many small GETs, where per-request + overhead dominates. Start with `--page-size 64` or larger. +- **Prefetch threshold.** A prefetch from SeaweedFS pays off only when loading the prefix is faster + than recomputing it, which depends on the model, the GPU, the prefix length and the bandwidth to + SeaweedFS. The default `prefetch_threshold` of 256 tokens is low for a remote store; measure + time-to-first-token with and without the cache for a few prefix lengths and set the threshold at + the break-even point, for example: + + ```bash + --hicache-storage-backend-extra-config '{"endpoint": "http://seaweedfs-s3:8333", "prefetch_threshold": 8192}' + ``` + + `prefetch_threshold`, `prefetch_timeout_base` and `prefetch_timeout_per_ki_token` are read by + SGLang's cache controller, not by this backend. + +## Object layout + +Objects are named `///`: + +- MHA models include `tpof`, because tensor-parallel ranks hold different heads. +- MLA models omit the tp rank, because every rank holds identical KV and only one rank backs up. +- Pipeline-parallel (`ppof`) and context-parallel (`cpof`) components are + added when those degrees are greater than one. + +Instances share cache entries only when the model name and parallel layout match. + +Hybrid models (Mamba, SWA, DeepSeek V4 and other side pools) store each side-pool page next to +its KV page as `.`. KV pages keep the same names on both interfaces, so one +cache serves both. + +## Behaviour and limits + +- The backend uses the zero-copy HiCache interface: pages move directly between SeaweedFS and the + host KV pool's own buffers, without a staging page. +- Hybrid models use the v2 interface (`batch_exists_v2`, `batch_get_v2`, `batch_set_v2`). A side + pool with the `trailing_pages` policy only needs the pages in its window to be present. +- A prefetch stops at the first missing page. Pages after a miss are not loaded. +- An object whose size does not match the host page is treated as a miss and never copied. +- `clear()` deletes only the calling rank's objects. Other models and ranks sharing the bucket are + left alone. +- The backend does not evict. Size the SeaweedFS cluster for the cache, or apply a retention + policy on the SeaweedFS side. diff --git a/python/sglang/srt/mem_cache/storage/seaweedfs/__init__.py b/python/sglang/srt/mem_cache/storage/seaweedfs/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/python/sglang/srt/mem_cache/storage/seaweedfs/seaweedfs_store.py b/python/sglang/srt/mem_cache/storage/seaweedfs/seaweedfs_store.py new file mode 100755 index 0000000000..608f653af3 --- /dev/null +++ b/python/sglang/srt/mem_cache/storage/seaweedfs/seaweedfs_store.py @@ -0,0 +1,431 @@ +"""SeaweedFS backend for HiCache L3 storage. + +Each KV page is one object in a SeaweedFS bucket, reached through SeaweedFS's S3 gateway. The +cache controller drives the zero-copy interface: ``batch_get_v1`` / ``batch_set_v1`` move a +page between its object and the host pool's own buffers (from ``get_page_buffer_meta``), +returning one success flag per page. An object holds the page's buffers back to back, which +for the page-first layouts is byte-identical to the flat page of the generic ``batch_get`` / +``batch_set`` interface, so the two interfaces read each other's objects. +``batch_exists`` returns the length of the leading run of existing keys. + +Configuration comes from ``--hicache-storage-backend-extra-config``; only ``endpoint`` is +required. When ``access_key``/``secret_key`` are absent the standard AWS credential chain is +used, which keeps secrets off the command line. +""" + +from __future__ import annotations + +import ctypes +import hashlib +import logging +from concurrent.futures import ThreadPoolExecutor +from typing import Any, Dict, List, Optional + +import msgspec +import torch + +from sglang.srt.mem_cache.hicache_storage import ( + HiCacheStorage, + HiCacheStorageConfig, + HiCacheStorageExtraInfo, + PoolHitPolicy, + PoolName, + PoolTransfer, + PoolTransferResult, +) + +logger = logging.getLogger(__name__) + +# S3 keys are capped at 1024 bytes; longer page keys are hashed down below this. +_MAX_KEY_LEN = 900 +_MISSING_CODES = ("404", "NoSuchKey", "NoSuchBucket", "NotFound") + + +class SeaweedFSConfig(msgspec.Struct, frozen=True, kw_only=True): + endpoint: str + bucket: str = "sglang-hicache" + prefix: str = "" + region: str = "us-east-1" + access_key: Optional[str] = None + secret_key: Optional[str] = None + max_workers: int = 16 + + @classmethod + def from_extra_config(cls, extra_config: Optional[dict]) -> SeaweedFSConfig: + # extra_config also carries factory keys (backend_name, module_path, ...). + fields = set(cls.__struct_fields__) + return msgspec.convert( + {k: v for k, v in (extra_config or {}).items() if k in fields}, type=cls + ) + + +def _make_s3_client(config: SeaweedFSConfig): + import boto3 + from botocore.config import Config as BotoConfig + + credentials = {} + if config.access_key and config.secret_key: + credentials = { + "aws_access_key_id": config.access_key, + "aws_secret_access_key": config.secret_key, + } + return boto3.client( + "s3", + endpoint_url=config.endpoint, + region_name=config.region, + # Path-style: virtual-hosted addressing needs per-bucket DNS a cluster gateway lacks. + config=BotoConfig( + s3={"addressing_style": "path"}, + max_pool_connections=max(config.max_workers, 10), + retries={"max_attempts": 3, "mode": "standard"}, + ), + **credentials, + ) + + +def key_scope(storage_config: HiCacheStorageConfig) -> List[str]: + """Model and rank components of an object key; mirrors HiCacheFile's suffix rules.""" + parts = [] + if storage_config.model_name: + parts.append("-".join(storage_config.model_name.split("/"))) + # MLA ranks hold identical KV and only one of them backs up, so they share keys; + # MHA ranks hold different heads and must not. + if not storage_config.is_mla_model: + parts.append(f"tp{storage_config.tp_rank}of{storage_config.tp_size}") + if storage_config.pp_size > 1: + parts.append(f"pp{storage_config.pp_rank}of{storage_config.pp_size}") + if storage_config.attn_cp_size > 1: + parts.append(f"cp{storage_config.attn_cp_rank}of{storage_config.attn_cp_size}") + return parts + + +def _byte_view(tensor: torch.Tensor) -> memoryview: + # Through uint8 so bf16/fp8 pages work; numpy has no such dtypes. + return memoryview(tensor.reshape(-1).view(torch.uint8).numpy()) + + +class SeaweedFSStore(HiCacheStorage): + def __init__( + self, storage_config: HiCacheStorageConfig, mem_pool_host: Any = None + ) -> None: + self.config = SeaweedFSConfig.from_extra_config(storage_config.extra_config) + scope = [self.config.prefix.strip("/")] if self.config.prefix else [] + self.key_prefix = "/".join(scope + key_scope(storage_config)) + self._s3 = _make_s3_client(self.config) + self._executor = ThreadPoolExecutor(max_workers=self.config.max_workers) + self._ensure_bucket() + logger.info( + "SeaweedFS HiCache backend: endpoint=%s bucket=%s prefix=%s", + self.config.endpoint, + self.config.bucket, + self.key_prefix, + ) + + def _is_missing(self, exc: Exception) -> bool: + return ( + isinstance(exc, self._s3.exceptions.ClientError) + and str(exc.response["Error"]["Code"]) in _MISSING_CODES + ) + + def _ensure_bucket(self) -> None: + try: + self._s3.head_bucket(Bucket=self.config.bucket) + except Exception as exc: + if not self._is_missing(exc): + raise + self._s3.create_bucket(Bucket=self.config.bucket) + + def _object_key(self, key: str) -> str: + full = f"{self.key_prefix}/{key}" if self.key_prefix else key + if len(full) <= _MAX_KEY_LEN: + return full + return f"{full[:800]}/{hashlib.sha256(full.encode()).hexdigest()}" + + def _fetch(self, key: str) -> Optional[bytes]: + try: + resp = self._s3.get_object( + Bucket=self.config.bucket, Key=self._object_key(key) + ) + return resp["Body"].read() + except Exception as exc: + if not self._is_missing(exc): + logger.warning("SeaweedFS get %s failed: %s", key, exc) + return None + + def get( + self, + key: str, + target_location: Optional[torch.Tensor] = None, + target_sizes: Optional[Any] = None, + ) -> torch.Tensor | None: + data = self._fetch(key) + if data is None: + return None + if target_location is None: + return torch.frombuffer(bytearray(data), dtype=torch.uint8) + nbytes = target_location.numel() * target_location.element_size() + # A size mismatch is a stale or foreign object; copying it would overrun the page. + if len(data) != nbytes or not target_location.is_contiguous(): + logger.warning( + "SeaweedFS object %s is %d bytes, page is %d", key, len(data), nbytes + ) + return None + _byte_view(target_location)[:] = data + return target_location + + def batch_get( + self, + keys: List[str], + target_locations: Optional[List[torch.Tensor]] = None, + target_sizes: Optional[Any] = None, + ) -> List[torch.Tensor | None]: + locations = target_locations or [None] * len(keys) + futures = [ + self._executor.submit(self.get, key, location) + for key, location in zip(keys, locations) + ] + return [f.result() for f in futures] + + def set( + self, + key: str, + value: Optional[torch.Tensor] = None, + target_location: Optional[torch.Tensor] = None, + target_sizes: Optional[Any] = None, + ) -> bool: + page = value if value is not None else target_location + try: + body = bytes(_byte_view(page.contiguous())) + self._s3.put_object( + Bucket=self.config.bucket, Key=self._object_key(key), Body=body + ) + return True + except Exception as exc: + logger.error("SeaweedFS set %s failed: %s", key, exc) + return False + + def batch_set( + self, + keys: List[str], + values: Optional[List[torch.Tensor]] = None, + target_locations: Optional[List[torch.Tensor]] = None, + target_sizes: Optional[Any] = None, + ) -> bool: + pages = values if values is not None else target_locations + futures = [ + self._executor.submit(self.set, key, page) for key, page in zip(keys, pages) + ] + return all([f.result() for f in futures]) + + def _page_buffers(self, host_indices: torch.Tensor) -> List[List[tuple]]: + """(address, nbytes) of each host-pool buffer of every page, in page order.""" + addresses, sizes = self.mem_pool_host.get_page_buffer_meta(host_indices) + num_pages = len(host_indices) // self.mem_pool_host.page_size + per_page = len(addresses) // num_pages + if isinstance(sizes, int): + sizes = [sizes] * len(addresses) + return [ + list( + zip( + addresses[i * per_page : (i + 1) * per_page], + sizes[i * per_page : (i + 1) * per_page], + ) + ) + for i in range(num_pages) + ] + + def _get_into(self, key: str, buffers: List[tuple]) -> bool: + data = self._fetch(key) + if data is None: + return False + expected = sum(nbytes for _, nbytes in buffers) + # A size mismatch is a stale or foreign object; copying it would overrun the page. + if len(data) != expected: + logger.warning( + "SeaweedFS object %s is %d bytes, page is %d", key, len(data), expected + ) + return False + src = ctypes.cast(ctypes.c_char_p(data), ctypes.c_void_p).value + offset = 0 + for address, nbytes in buffers: + ctypes.memmove(address, src + offset, nbytes) + offset += nbytes + return True + + def _put_from(self, key: str, buffers: List[tuple]) -> bool: + try: + body = bytearray(sum(nbytes for _, nbytes in buffers)) + dst = ctypes.addressof(ctypes.c_char.from_buffer(body)) + offset = 0 + for address, nbytes in buffers: + ctypes.memmove(dst + offset, address, nbytes) + offset += nbytes + self._s3.put_object( + Bucket=self.config.bucket, Key=self._object_key(key), Body=body + ) + return True + except Exception as exc: + logger.error("SeaweedFS set %s failed: %s", key, exc) + return False + + def batch_get_v1( + self, + keys: List[str], + host_indices: torch.Tensor, + extra_info: Optional[HiCacheStorageExtraInfo] = None, + ) -> List[bool]: + pages = self._page_buffers(host_indices) + futures = [ + self._executor.submit(self._get_into, key, buffers) + for key, buffers in zip(keys, pages) + ] + return [f.result() for f in futures] + + def batch_set_v1( + self, + keys: List[str], + host_indices: torch.Tensor, + extra_info: Optional[HiCacheStorageExtraInfo] = None, + ) -> List[bool]: + pages = self._page_buffers(host_indices) + futures = [ + self._executor.submit(self._put_from, key, buffers) + for key, buffers in zip(keys, pages) + ] + return [f.result() for f in futures] + + # Hybrid models (Mamba, SWA, DeepSeek V4 indexer and other side pools) go + # through the v2 interface. Side-pool pages are stored next to their KV page + # as "."; KV pages keep the same object names as v1, so a cache + # written by either path is readable by the other. + + def _component_key(self, key: str, pool_name) -> str: + name = getattr(pool_name, "value", pool_name) + return key if name == PoolName.KV.value else f"{key}.{name}" + + def batch_exists_v2( + self, + keys: List[str], + pool_transfers: Optional[List[PoolTransfer]] = None, + extra_info: Optional[HiCacheStorageExtraInfo] = None, + ) -> PoolTransferResult: + kv_pages = self.batch_exists(keys, extra_info) + hit_count: Dict[str, int] = {PoolName.KV: kv_pages} if kv_pages else {} + final_pages = kv_pages + for transfer in pool_transfers or []: + if final_pages == 0: + break + futures = [ + self._executor.submit( + self.exists, self._component_key(keys[i], transfer.name) + ) + for i in range(kv_pages) + ] + present = [f.result() for f in futures] + if transfer.hit_policy == PoolHitPolicy.ALL_PAGES: + boundary = next((i for i, ok in enumerate(present) if not ok), kv_pages) + else: # TRAILING_PAGES: only the window ending at the prefix must exist + trailing = max(1, len(transfer.keys) if transfer.keys else 1) + boundary = next( + ( + n + for n in range(kv_pages, 0, -1) + if all(present[max(0, n - trailing) : n]) + ), + 0, + ) + if boundary: + hit_count[transfer.name] = boundary + final_pages = min(final_pages, boundary) + return PoolTransferResult(final_pages, hit_count) + + def _read_page_v2(self, pool_name, key: str, host_pool, offset: int) -> bool: + page = host_pool.get_dummy_flat_data_page() + if self.get(self._component_key(key, pool_name), page) is None: + return False + host_pool.set_from_flat_data_page(offset, page) + return True + + def _write_page_v2(self, pool_name, key: str, host_pool, offset: int) -> bool: + page = host_pool.get_data_page(offset, flat=True) + return self.set(self._component_key(key, pool_name), page) + + def _batch_io_v2(self, transfers: List[PoolTransfer], page_fn): + results: Dict[str, List[bool]] = {} + pending = [] + for transfer in transfers: + keys = transfer.keys or [] + host_pool = getattr(self, "registered_pools", {}).get(transfer.name) + if host_pool is None: + logger.error("SeaweedFS: host pool %s is not registered", transfer.name) + results[transfer.name] = [False] * len(keys) + continue + page_size = getattr(host_pool, "page_size", 1) or 1 + indices = transfer.host_indices + if indices is None or indices.numel() != len(keys) * page_size: + logger.error( + "SeaweedFS: %s has %d keys but %s host indices (page size %d)", + transfer.name, + len(keys), + indices.numel() if indices is not None else 0, + page_size, + ) + results[transfer.name] = [False] * len(keys) + continue + futures = [ + self._executor.submit( + page_fn, + transfer.name, + key, + host_pool, + indices[i * page_size].item(), + ) + for i, key in enumerate(keys) + ] + pending.append((transfer.name, futures)) + for name, futures in pending: + results[name] = [f.result() for f in futures] + return results + + def batch_get_v2( + self, + transfers: List[PoolTransfer], + extra_info: Optional[HiCacheStorageExtraInfo] = None, + ) -> Dict[str, List[bool]]: + return self._batch_io_v2(transfers, self._read_page_v2) + + def batch_set_v2( + self, + transfers: List[PoolTransfer], + extra_info: Optional[HiCacheStorageExtraInfo] = None, + ) -> Dict[str, List[bool]]: + return self._batch_io_v2(transfers, self._write_page_v2) + + def exists(self, key: str) -> bool: + try: + self._s3.head_object(Bucket=self.config.bucket, Key=self._object_key(key)) + return True + except Exception as exc: + if not self._is_missing(exc): + logger.warning("SeaweedFS exists %s failed: %s", key, exc) + return False + + def batch_exists( + self, keys: List[str], extra_info: Optional[HiCacheStorageExtraInfo] = None + ) -> int: + futures = [self._executor.submit(self.exists, key) for key in keys] + for i, f in enumerate(futures): + if not f.result(): + return i + return len(keys) + + def clear(self) -> None: + # Only this rank's prefix; other models and ranks may share the bucket. + prefix = f"{self.key_prefix}/" if self.key_prefix else "" + paginator = self._s3.get_paginator("list_objects_v2") + for page in paginator.paginate(Bucket=self.config.bucket, Prefix=prefix): + objects = [{"Key": o["Key"]} for o in page.get("Contents", [])] + if objects: + self._s3.delete_objects( + Bucket=self.config.bucket, Delete={"Objects": objects} + ) diff --git a/test/registered/unit/mem_cache/test_hicache_seaweedfs_storage.py b/test/registered/unit/mem_cache/test_hicache_seaweedfs_storage.py new file mode 100755 index 0000000000..9195cf5151 --- /dev/null +++ b/test/registered/unit/mem_cache/test_hicache_seaweedfs_storage.py @@ -0,0 +1,405 @@ +"""Unit tests for the SeaweedFS HiCache storage backend. + +Pages move through the cache controller's own generic get/set paths and a real CPU host pool; +only the S3 client is replaced, by an in-memory store with S3's miss semantics. + +Run with: + python3 -m pytest test/registered/unit/mem_cache/test_hicache_seaweedfs_storage.py -v +""" + +from sglang.test.ci.ci_register import register_cpu_ci + +register_cpu_ci(est_time=10, suite="base-a-test-cpu") + +import io +import unittest +import uuid +from types import SimpleNamespace +from unittest import mock + +import torch + +from sglang.srt.managers.cache_controller import HiCacheController +from sglang.srt.mem_cache.hicache_storage import ( + HiCacheStorageConfig, + HiCacheStorageExtraInfo, + PoolHitPolicy, + PoolName, + PoolTransfer, +) +from sglang.srt.mem_cache.pool_host.mha import MHATokenToKVPoolHost +from sglang.srt.mem_cache.storage import backend_factory +from sglang.srt.mem_cache.storage.seaweedfs import seaweedfs_store +from sglang.test.test_utils import CustomTestCase + +PAGE_SIZE = 16 +NUM_PAGES = 4 + + +class _ClientError(Exception): + def __init__(self, code: str): + super().__init__(code) + self.response = {"Error": {"Code": code}} + + +class InMemoryS3: + """The subset of the boto3 S3 client the backend uses, with S3's miss behaviour.""" + + exceptions = SimpleNamespace(ClientError=_ClientError) + + def __init__(self, buckets: dict): + self.buckets = buckets + + def _bucket(self, bucket): + if bucket not in self.buckets: + raise _ClientError("NoSuchBucket") + return self.buckets[bucket] + + def head_bucket(self, Bucket): + self._bucket(Bucket) + + def create_bucket(self, Bucket): + self.buckets.setdefault(Bucket, {}) + + def put_object(self, Bucket, Key, Body): + self._bucket(Bucket)[Key] = bytes(Body) + + def get_object(self, Bucket, Key): + objects = self._bucket(Bucket) + if Key not in objects: + raise _ClientError("NoSuchKey") + return {"Body": io.BytesIO(objects[Key])} + + def head_object(self, Bucket, Key): + if Key not in self._bucket(Bucket): + raise _ClientError("404") + + def delete_object(self, Bucket, Key): + self._bucket(Bucket).pop(Key, None) + + def delete_objects(self, Bucket, Delete): + for obj in Delete["Objects"]: + self._bucket(Bucket).pop(obj["Key"], None) + + def get_paginator(self, name): + store = self + + class _Paginator: + def paginate(self, Bucket, Prefix): + keys = [k for k in store._bucket(Bucket) if k.startswith(Prefix)] + yield {"Contents": [{"Key": k} for k in keys]} + + return _Paginator() + + +def _host_pool(layout: str) -> MHATokenToKVPoolHost: + device_pool = SimpleNamespace( + store_dtype=torch.bfloat16, + size=64, + layer_num=4, + start_layer=0, + end_layer=4, + head_num=2, + head_dim=64, + v_head_dim=64, + row_dim=128, + device="cpu", + layer_shard_enabled=False, + hicache_write_back_staging=None, + ) + return MHATokenToKVPoolHost( + device_pool=device_pool, + host_to_device_ratio=2.0, + host_size=0, + page_size=PAGE_SIZE, + layout=layout, + pin_memory=False, + device="cpu", + ) + + +def _page(pool: MHATokenToKVPoolHost, page: int) -> torch.Tensor: + return pool.get_data_page(page * PAGE_SIZE, flat=True).view(torch.uint8).clone() + + +class TestSeaweedFSStore(CustomTestCase): + endpoint = "http://seaweedfs.invalid:8333" + + def make_client(self, config): + return InMemoryS3(self.buckets) + + def setUp(self): + self.buckets = {} + self.bucket = f"sgl-{uuid.uuid4().hex[:10]}" + patcher = mock.patch.object( + seaweedfs_store, + "_make_s3_client", + side_effect=lambda config: self.make_client(config), + ) + patcher.start() + self.addCleanup(patcher.stop) + + def _config(self, tp_rank=0, tp_size=1, is_mla=False, cp_rank=0, cp_size=1): + return HiCacheStorageConfig( + tp_rank=tp_rank, + tp_size=tp_size, + pp_rank=0, + pp_size=1, + attn_cp_rank=cp_rank, + attn_cp_size=cp_size, + is_mla_model=is_mla, + enable_storage_metrics=False, + is_page_first_layout=True, + model_name="Qwen/Qwen3-0.6B", + extra_config={"endpoint": self.endpoint, "bucket": self.bucket}, + ) + + def _store(self, **kwargs): + return backend_factory.StorageBackendFactory.create_backend( + backend_name="seaweedfs", + storage_config=self._config(**kwargs), + mem_pool_host=None, + ) + + def _controller(self, store, pool): + store.register_mem_pool_host(pool) + return SimpleNamespace( + storage_backend=store, storage_host_pool=pool, page_size=PAGE_SIZE + ) + + def _backup_and_reload( + self, store, layout, drop_page=None, set_via="generic", get_via="generic" + ): + """Back up NUM_PAGES random pages, wipe the pool, prefetch them back. + + set_via / get_via pick the controller path: "generic" (flat page tensors) or + "zero_copy" (batch_get_v1 / batch_set_v1 on the host pool's own buffers). + """ + pool = _host_pool(layout) + ctrl = self._controller(store, pool) + pool.kv_buffer.copy_(torch.randn(pool.kv_buffer.shape).to(torch.bfloat16)) + expected = [_page(pool, i) for i in range(NUM_PAGES)] + keys = [f"{layout}-{uuid.uuid4().hex[:8]}-{i}" for i in range(NUM_PAGES)] + host_indices = torch.arange(NUM_PAGES * PAGE_SIZE, dtype=torch.int64) + + page_set = { + "generic": HiCacheController._generic_page_set, + "zero_copy": HiCacheController._page_set_zero_copy, + }[set_via] + page_get = { + "generic": HiCacheController._generic_page_get, + "zero_copy": HiCacheController._page_get_zero_copy, + }[get_via] + self.assertTrue(page_set(ctrl, keys, host_indices)) + if drop_page is not None: + store._s3.delete_object( + Bucket=self.bucket, Key=store._object_key(keys[drop_page]) + ) + pool.kv_buffer.zero_() + operation = SimpleNamespace(request_id="r", is_terminated=lambda: False) + loaded = page_get(ctrl, operation, keys, host_indices) + return pool, keys, expected, loaded + + def test_controller_round_trip_is_bit_exact_for_every_layout(self): + store = self._store() + for via in ("generic", "zero_copy"): + for layout in ("layer_first", "page_first", "page_first_direct"): + with self.subTest(via=via, layout=layout): + pool, _, expected, loaded = self._backup_and_reload( + store, layout, set_via=via, get_via=via + ) + self.assertEqual(loaded, NUM_PAGES) + for i in range(NUM_PAGES): + self.assertTrue(torch.equal(_page(pool, i), expected[i])) + + def test_generic_and_zero_copy_read_each_others_pages(self): + """Both interfaces store a page as one object with the same bytes (page-first layouts).""" + store = self._store() + for layout in ("page_first", "page_first_direct"): + for set_via, get_via in ( + ("generic", "zero_copy"), + ("zero_copy", "generic"), + ): + with self.subTest(layout=layout, set_via=set_via): + pool, _, expected, loaded = self._backup_and_reload( + store, layout, set_via=set_via, get_via=get_via + ) + self.assertEqual(loaded, NUM_PAGES) + for i in range(NUM_PAGES): + self.assertTrue(torch.equal(_page(pool, i), expected[i])) + + def test_prefetch_stops_at_the_first_missing_page(self): + """Pages after a miss are unusable and must not be loaded or counted.""" + store = self._store() + for via in ("generic", "zero_copy"): + with self.subTest(via=via): + pool, keys, _, loaded = self._backup_and_reload( + store, "page_first", drop_page=2, set_via=via, get_via=via + ) + self.assertEqual(loaded, 2) + if via == "generic": + # Zero-copy fetches a batch in parallel into slots the controller + # releases after the count; only the generic path skips the copy. + self.assertFalse(_page(pool, 3).any()) + extra_info = HiCacheStorageExtraInfo(prefix_keys=[]) + self.assertEqual(store.batch_exists(keys, extra_info), 2) + + def test_zero_copy_rejects_an_object_of_the_wrong_size(self): + """A stale or foreign object must not be copied into the host pool.""" + store = self._store() + pool = _host_pool("page_first") + pool.kv_buffer.zero_() + store.register_mem_pool_host(pool) + store._s3.put_object( + Bucket=self.bucket, Key=store._object_key("k"), Body=b"\x7f" * 8192 + ) + host_indices = torch.arange(PAGE_SIZE, dtype=torch.int64) + self.assertEqual(store.batch_get_v1(["k"], host_indices), [False]) + self.assertFalse(pool.kv_buffer.any()) + + def test_object_of_the_wrong_size_is_a_miss_not_an_overrun(self): + store = self._store() + store._s3.put_object( + Bucket=self.bucket, Key=store._object_key("k"), Body=b"\x7f" * 8192 + ) + arena = torch.zeros(8192, dtype=torch.uint8) + self.assertIsNone(store.get("k", arena[:4096])) + self.assertFalse(arena.any()) + + def test_mha_and_context_parallel_ranks_do_not_share_objects(self): + for kwargs in ( + [dict(tp_rank=0, tp_size=2), dict(tp_rank=1, tp_size=2)], + [dict(cp_rank=0, cp_size=2), dict(cp_rank=1, cp_size=2)], + ): + with self.subTest(ranks=kwargs): + first, second = self._store(**kwargs[0]), self._store(**kwargs[1]) + first.set("shared", torch.full((64,), 1, dtype=torch.uint8)) + second.set("shared", torch.full((64,), 2, dtype=torch.uint8)) + self.assertEqual(int(first.get("shared")[0]), 1) + second.clear() + self.assertTrue(first.exists("shared")) + + def test_mla_ranks_share_objects(self): + """MLA ranks hold identical KV and only one backs up, so the others must find it.""" + writer = self._store(tp_rank=0, tp_size=2, is_mla=True) + reader = self._store(tp_rank=1, tp_size=2, is_mla=True) + writer.set("shared", torch.full((64,), 3, dtype=torch.uint8)) + self.assertTrue(reader.exists("shared")) + + # ---- v2 interface (hybrid models): KV plus a side pool ----------------- + # A second real host pool stands in for the side pool (Mamba/SWA/indexer): + # the backend only touches it through the host-pool page API, so what this + # proves is the per-pool object naming, routing and hit policies. + + def _v2_store(self, layout="page_first"): + store = self._store() + kv, side = _host_pool(layout), _host_pool(layout) + store.register_mem_host_pool_v2(kv, PoolName.KV) + store.register_mem_host_pool_v2(side, PoolName.MAMBA) + for pool in (kv, side): + pool.kv_buffer.copy_(torch.randn(pool.kv_buffer.shape).to(torch.bfloat16)) + keys = [f"v2-{uuid.uuid4().hex[:8]}-{i}" for i in range(NUM_PAGES)] + indices = torch.arange(NUM_PAGES * PAGE_SIZE, dtype=torch.int64) + transfers = [ + PoolTransfer(name=PoolName.KV, host_indices=indices, keys=keys), + PoolTransfer(name=PoolName.MAMBA, host_indices=indices, keys=keys), + ] + return store, kv, side, keys, transfers + + def test_v2_round_trip_is_bit_exact_per_pool(self): + for layout in ("layer_first", "page_first", "page_first_direct"): + with self.subTest(layout=layout): + store, kv, side, keys, transfers = self._v2_store(layout) + expected = { + p: [_page(pool, i) for i in range(NUM_PAGES)] + for p, pool in ((PoolName.KV, kv), (PoolName.MAMBA, side)) + } + written = store.batch_set_v2(transfers) + self.assertEqual(written[PoolName.KV], [True] * NUM_PAGES) + self.assertEqual(written[PoolName.MAMBA], [True] * NUM_PAGES) + kv.kv_buffer.zero_() + side.kv_buffer.zero_() + loaded = store.batch_get_v2(transfers) + for p, pool in ((PoolName.KV, kv), (PoolName.MAMBA, side)): + self.assertEqual(loaded[p], [True] * NUM_PAGES) + for i in range(NUM_PAGES): + self.assertTrue(torch.equal(_page(pool, i), expected[p][i])) + + def test_v2_kv_pages_are_the_v1_objects(self): + # KV pages keep their v1 object names, so v1 and v2 share one cache; + # side-pool pages are separate objects and never overwrite them. + store, kv, side, keys, transfers = self._v2_store() + store.register_mem_pool_host(kv) # the v1 path reads through this pool + store.batch_set_v2(transfers) + self.assertEqual(store.batch_exists(keys), NUM_PAGES) + self.assertEqual(store.batch_exists([f"{k}.mamba" for k in keys]), NUM_PAGES) + self.assertEqual( + store.batch_get_v1(keys, transfers[0].host_indices), [True] * NUM_PAGES + ) + + def test_v2_exists_all_pages_stops_at_a_missing_side_page(self): + store, _, _, keys, transfers = self._v2_store() + store.batch_set_v2(transfers) + store._s3.delete_object( + Bucket=self.bucket, Key=store._object_key(f"{keys[2]}.mamba") + ) + side = PoolTransfer( + name=PoolName.MAMBA, keys=keys, hit_policy=PoolHitPolicy.ALL_PAGES + ) + result = store.batch_exists_v2(keys, [side]) + self.assertEqual(result.kv_hit_pages, 2) + self.assertEqual( + result.extra_pool_hit_pages, {PoolName.KV: NUM_PAGES, PoolName.MAMBA: 2} + ) + + def test_v2_exists_trailing_pages_needs_only_the_window(self): + store, _, _, keys, transfers = self._v2_store() + store.batch_set_v2(transfers) + # Only the last page's side state matters for a one-page window, so a + # gap earlier in the side pool does not shorten the prefix... + store._s3.delete_object( + Bucket=self.bucket, Key=store._object_key(f"{keys[0]}.mamba") + ) + window = PoolTransfer( + name=PoolName.MAMBA, keys=keys[-1:], hit_policy=PoolHitPolicy.TRAILING_PAGES + ) + self.assertEqual(store.batch_exists_v2(keys, [window]).kv_hit_pages, NUM_PAGES) + # ...but losing the last page's state falls back to the longest prefix + # whose own last page still has it. + store._s3.delete_object( + Bucket=self.bucket, Key=store._object_key(f"{keys[-1]}.mamba") + ) + result = store.batch_exists_v2(keys, [window]) + self.assertEqual(result.kv_hit_pages, NUM_PAGES - 1) + self.assertEqual(result.extra_pool_hit_pages[PoolName.MAMBA], NUM_PAGES - 1) + + def test_v2_exists_without_kv_prefix_is_empty(self): + store, _, _, keys, _ = self._v2_store() + side = PoolTransfer(name=PoolName.MAMBA, keys=keys) + result = store.batch_exists_v2(keys, [side]) + self.assertEqual((result.kv_hit_pages, result.extra_pool_hit_pages), (0, {})) + + def test_v2_rejects_mismatched_indices_and_unknown_pools(self): + store, _, _, keys, _ = self._v2_store() + short = torch.arange(PAGE_SIZE, dtype=torch.int64) # one page for four keys + bad = [ + PoolTransfer(name=PoolName.KV, host_indices=short, keys=keys), + PoolTransfer( + name=PoolName.SWA, + host_indices=torch.arange(NUM_PAGES * PAGE_SIZE), + keys=keys, + ), + ] + for op in (store.batch_set_v2, store.batch_get_v2): + result = op(bad) + self.assertEqual(result[PoolName.KV], [False] * NUM_PAGES) + self.assertEqual(result[PoolName.SWA], [False] * NUM_PAGES) + self.assertEqual(store.batch_exists(keys), 0) + + def test_missing_endpoint_is_refused(self): + with self.assertRaises(ValueError): + seaweedfs_store.SeaweedFSConfig.from_extra_config({"bucket": "b"}) + + +if __name__ == "__main__": + unittest.main()