[MemCache] Serve disabled pure-SWA models with PureSWARadixCache (#43198)
This commit is contained in:
@@ -16,7 +16,8 @@ class PureSWARadixCache(RadixCache):
|
||||
|
||||
Extends RadixCache with SWA semantics. Only caches the prefill portion
|
||||
[0, evict_floor) on request completion. Window-range KV is freed.
|
||||
No tombstone mechanism needed.
|
||||
No tombstone mechanism needed. With ``params.disable`` it serves
|
||||
``--disable-radix-cache`` and caches nothing.
|
||||
"""
|
||||
|
||||
def __init__(self, params: CacheInitParams):
|
||||
|
||||
@@ -93,15 +93,17 @@ def default_radix_cache_factory(ctx: TreeCacheBuildContext) -> BasePrefixCache:
|
||||
):
|
||||
return create_unified_radix_cache(ctx)
|
||||
|
||||
# A disabled cache publishes nothing, so no storage backend applies.
|
||||
if ctx.disable_radix_cache and is_pure_swa:
|
||||
from sglang.srt.mem_cache.pure_swa_radix_cache import PureSWARadixCache
|
||||
|
||||
return PureSWARadixCache(params=params)
|
||||
|
||||
if ctx.effective_chunked_prefill_size is not None and ctx.disable_radix_cache:
|
||||
if not ctx.is_hybrid_swa:
|
||||
from sglang.srt.mem_cache.chunk_cache import ChunkCache
|
||||
|
||||
return ChunkCache(params)
|
||||
if ctx.full_tokens_per_layer == 0:
|
||||
from sglang.srt.mem_cache.chunk_cache import PureSWAChunkCache
|
||||
|
||||
return PureSWAChunkCache(params)
|
||||
from sglang.srt.mem_cache.chunk_cache import SWAChunkCache
|
||||
|
||||
return SWAChunkCache(params)
|
||||
|
||||
Reference in New Issue
Block a user