[MemCache] Serve disabled hybrid-SWA models with UnifiedRadixCache (#43200)
This commit is contained in:
@@ -82,15 +82,11 @@ def default_radix_cache_factory(ctx: TreeCacheBuildContext) -> BasePrefixCache:
|
||||
params = ctx.params
|
||||
|
||||
is_pure_swa = ctx.is_hybrid_swa and ctx.full_tokens_per_layer == 0
|
||||
# The decode host-pool backup needs UnifiedRadixCache; otherwise its
|
||||
# disabled mode serves every layout but pure-SWA.
|
||||
if ctx.disable_radix_cache and (
|
||||
get_disagg().disaggregation_decode_retraction_backup == "host_pool"
|
||||
# UnifiedRadixCache's disabled mode serves full-attention layouts,
|
||||
# streaming sessions, and mamba states; pure-SWA has no unified layout.
|
||||
or not ctx.is_hybrid_swa
|
||||
or (
|
||||
not is_pure_swa
|
||||
and (get_serving().enable_streaming_session or ctx.is_hybrid_ssm)
|
||||
)
|
||||
or not is_pure_swa
|
||||
):
|
||||
return create_unified_radix_cache(ctx)
|
||||
|
||||
@@ -100,11 +96,6 @@ def default_radix_cache_factory(ctx: TreeCacheBuildContext) -> BasePrefixCache:
|
||||
|
||||
return PureSWARadixCache(params=params)
|
||||
|
||||
if ctx.effective_chunked_prefill_size is not None and ctx.disable_radix_cache:
|
||||
from sglang.srt.mem_cache.chunk_cache import SWAChunkCache
|
||||
|
||||
return SWAChunkCache(params)
|
||||
|
||||
if get_memory().enable_lmcache:
|
||||
from sglang.srt.mem_cache.storage.lmcache.lmcache_unified_radix_cache import (
|
||||
LMCacheUnifiedRadixCache,
|
||||
|
||||
@@ -69,12 +69,19 @@ class SWAComponent(TreeComponent):
|
||||
"""
|
||||
|
||||
def __init__(self, cache: UnifiedRadixCache, params: CacheInitParams):
|
||||
from sglang.srt.mem_cache.allocator.hisparse import (
|
||||
DeepSeekV4HiSparseTokenToKVPoolAllocator,
|
||||
)
|
||||
from sglang.srt.mem_cache.allocator.swa import SWATokenToKVPoolAllocator
|
||||
|
||||
# DeepSeek V4 HiSparse forces --disable-radix-cache, where this component
|
||||
# only frees through free_swa_segment / free / free_group_*, which the
|
||||
# HiSparse allocator defines.
|
||||
assert isinstance(
|
||||
params.token_to_kv_pool_allocator, SWATokenToKVPoolAllocator
|
||||
params.token_to_kv_pool_allocator,
|
||||
(SWATokenToKVPoolAllocator, DeepSeekV4HiSparseTokenToKVPoolAllocator),
|
||||
), (
|
||||
f"SWAComponent requires SWATokenToKVPoolAllocator, got {type(params.token_to_kv_pool_allocator)}"
|
||||
f"SWAComponent requires an SWA allocator, got {type(params.token_to_kv_pool_allocator)}"
|
||||
)
|
||||
if params.sliding_window_size is None or params.sliding_window_size <= 0:
|
||||
raise ValueError("SWAComponent requires a positive sliding_window_size")
|
||||
|
||||
Reference in New Issue
Block a user