Tiny adjust hybrid swa handling (#16292)
This commit is contained in:
@@ -63,7 +63,6 @@ from sglang.srt.mem_cache.allocator import (
|
|||||||
SWATokenToKVPoolAllocator,
|
SWATokenToKVPoolAllocator,
|
||||||
)
|
)
|
||||||
from sglang.srt.mem_cache.base_prefix_cache import BasePrefixCache
|
from sglang.srt.mem_cache.base_prefix_cache import BasePrefixCache
|
||||||
from sglang.srt.mem_cache.chunk_cache import SWAChunkCache
|
|
||||||
from sglang.srt.mem_cache.common import (
|
from sglang.srt.mem_cache.common import (
|
||||||
alloc_for_decode,
|
alloc_for_decode,
|
||||||
alloc_for_extend,
|
alloc_for_extend,
|
||||||
@@ -73,7 +72,6 @@ from sglang.srt.mem_cache.common import (
|
|||||||
from sglang.srt.mem_cache.mamba_radix_cache import MambaRadixCache
|
from sglang.srt.mem_cache.mamba_radix_cache import MambaRadixCache
|
||||||
from sglang.srt.mem_cache.memory_pool import ReqToTokenPool
|
from sglang.srt.mem_cache.memory_pool import ReqToTokenPool
|
||||||
from sglang.srt.mem_cache.radix_cache import RadixKey
|
from sglang.srt.mem_cache.radix_cache import RadixKey
|
||||||
from sglang.srt.mem_cache.swa_radix_cache import SWARadixCache
|
|
||||||
from sglang.srt.metrics.collector import (
|
from sglang.srt.metrics.collector import (
|
||||||
DPCooperationInfo,
|
DPCooperationInfo,
|
||||||
SchedulerMetricsCollector,
|
SchedulerMetricsCollector,
|
||||||
@@ -1299,11 +1297,6 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin):
|
|||||||
|
|
||||||
is_hybrid_swa = False
|
is_hybrid_swa = False
|
||||||
if isinstance(token_to_kv_pool_allocator, SWATokenToKVPoolAllocator):
|
if isinstance(token_to_kv_pool_allocator, SWATokenToKVPoolAllocator):
|
||||||
assert (
|
|
||||||
tree_cache is None
|
|
||||||
or isinstance(tree_cache, SWARadixCache)
|
|
||||||
or isinstance(tree_cache, SWAChunkCache)
|
|
||||||
), "SWARadixCache or SWAChunkCache is required for SWATokenToKVPoolAllocator"
|
|
||||||
is_hybrid_swa = True
|
is_hybrid_swa = True
|
||||||
|
|
||||||
return cls(
|
return cls(
|
||||||
|
|||||||
@@ -30,6 +30,7 @@ from sglang.srt.mem_cache.allocator import SWATokenToKVPoolAllocator
|
|||||||
from sglang.srt.mem_cache.base_prefix_cache import BasePrefixCache
|
from sglang.srt.mem_cache.base_prefix_cache import BasePrefixCache
|
||||||
from sglang.srt.mem_cache.mamba_radix_cache import MambaRadixCache
|
from sglang.srt.mem_cache.mamba_radix_cache import MambaRadixCache
|
||||||
from sglang.srt.mem_cache.radix_cache import RadixCache, RadixKey, TreeNode
|
from sglang.srt.mem_cache.radix_cache import RadixCache, RadixKey, TreeNode
|
||||||
|
from sglang.srt.mem_cache.swa_radix_cache import SWARadixCache
|
||||||
from sglang.srt.server_args import ServerArgs
|
from sglang.srt.server_args import ServerArgs
|
||||||
|
|
||||||
if TYPE_CHECKING:
|
if TYPE_CHECKING:
|
||||||
@@ -472,17 +473,13 @@ class PrefillAdder:
|
|||||||
|
|
||||||
@contextmanager
|
@contextmanager
|
||||||
def _lock_node(self, last_node: TreeNode):
|
def _lock_node(self, last_node: TreeNode):
|
||||||
if self.is_hybrid_swa:
|
try:
|
||||||
try:
|
ret = self.tree_cache.inc_lock_ref(last_node)
|
||||||
swa_uuid_for_lock = self.tree_cache.inc_lock_ref(last_node)
|
yield None
|
||||||
yield None
|
finally:
|
||||||
finally:
|
if isinstance(self.tree_cache, SWARadixCache):
|
||||||
self.tree_cache.dec_lock_ref(last_node, swa_uuid_for_lock)
|
self.tree_cache.dec_lock_ref(last_node, ret)
|
||||||
else:
|
else:
|
||||||
try:
|
|
||||||
self.tree_cache.inc_lock_ref(last_node)
|
|
||||||
yield None
|
|
||||||
finally:
|
|
||||||
self.tree_cache.dec_lock_ref(last_node)
|
self.tree_cache.dec_lock_ref(last_node)
|
||||||
|
|
||||||
def add_one_req_ignore_eos(self, req: Req):
|
def add_one_req_ignore_eos(self, req: Req):
|
||||||
|
|||||||
Reference in New Issue
Block a user