[HiCache] Keep storage workers alive on backend exceptions (#43550)
This commit is contained in:
@@ -43,7 +43,7 @@ from sglang.srt.layers.dp_attention import (
|
||||
from sglang.srt.mem_cache.l2_transfer import L2Transfer, L2TransferEngine
|
||||
from sglang.srt.mem_cache.memory_pool import MLATokenToKVPool
|
||||
from sglang.srt.mem_cache.utils import get_storage_hash_str
|
||||
from sglang.srt.runtime_context import get_memory, get_parallel
|
||||
from sglang.srt.runtime_context import get_parallel
|
||||
from sglang.srt.utils import get_device_module
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -1126,8 +1126,6 @@ class HiCacheController:
|
||||
kv_derived_transfers,
|
||||
)
|
||||
except Exception:
|
||||
if not get_memory().enable_unified_memory:
|
||||
raise
|
||||
logger.exception(
|
||||
"HiCache prefetch transfer failed for request %s",
|
||||
operation.request_id,
|
||||
@@ -1288,8 +1286,6 @@ class HiCacheController:
|
||||
operation
|
||||
)
|
||||
except Exception:
|
||||
if not get_memory().enable_unified_memory:
|
||||
raise
|
||||
logger.exception(
|
||||
"HiCache storage query failed for request %s",
|
||||
operation.request_id,
|
||||
@@ -1374,6 +1370,14 @@ class HiCacheController:
|
||||
prefix_keys = prefix_keys + batch_hashes
|
||||
operation.completed_tokens += self.page_size * len(batch_hashes)
|
||||
|
||||
def _page_backup_or_log(self, operation):
|
||||
try:
|
||||
self._page_backup(operation)
|
||||
except Exception:
|
||||
logger.exception(
|
||||
"HiCache backup failed after %d tokens", operation.completed_tokens
|
||||
)
|
||||
|
||||
def backup_thread_func(self):
|
||||
"""
|
||||
Manage backup operations from host memory to storage backend.
|
||||
@@ -1385,7 +1389,7 @@ class HiCacheController:
|
||||
continue
|
||||
|
||||
if not self.backup_skip:
|
||||
self._page_backup(operation)
|
||||
self._page_backup_or_log(operation)
|
||||
self.ack_backup_queue.put(operation)
|
||||
|
||||
except Empty:
|
||||
|
||||
@@ -41,7 +41,6 @@ from sglang.srt.mem_cache.pool_host.base import uses_shared_host_layout
|
||||
from sglang.srt.mem_cache.pool_host.mha import MHATokenToKVPoolHost
|
||||
from sglang.srt.mem_cache.pool_host.unified import UnifiedPageEnvelopeHostPool
|
||||
from sglang.srt.mem_cache.radix_cache import RadixKey
|
||||
from sglang.srt.runtime_context import get_memory
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from sglang.srt.mem_cache.allocator import BaseTokenToKVPoolAllocator
|
||||
@@ -1513,8 +1512,6 @@ class HybridCacheController(BaseHiCacheController):
|
||||
)
|
||||
pool_hits = count_pool_hits(results)
|
||||
except Exception:
|
||||
if not get_memory().enable_unified_memory:
|
||||
raise
|
||||
logger.exception(
|
||||
"HiCache sidecar prefetch failed for request %s",
|
||||
operation.request_id,
|
||||
@@ -1630,7 +1627,7 @@ class HybridCacheController(BaseHiCacheController):
|
||||
operation = self.backup_queue.get(block=True, timeout=1)
|
||||
if operation is None:
|
||||
continue
|
||||
self._page_backup(operation)
|
||||
self._page_backup_or_log(operation)
|
||||
self.ack_backup_queue.put(operation)
|
||||
except Empty:
|
||||
continue
|
||||
|
||||
@@ -1122,7 +1122,7 @@ def _build_unified_host_pair(bundle):
|
||||
|
||||
|
||||
class TestUnifiedPageEnvelopeHostPool(CustomTestCase):
|
||||
def test_sidecar_read_error_is_only_a_cache_miss_with_unified_memory(self):
|
||||
def test_sidecar_read_error_is_a_cache_miss_with_unified_memory(self):
|
||||
from sglang.srt.runtime_context import publish, reset_context
|
||||
from sglang.srt.server_args import ServerArgs
|
||||
|
||||
@@ -1134,34 +1134,28 @@ class TestUnifiedPageEnvelopeHostPool(CustomTestCase):
|
||||
raise ValueError("sidecar read failed")
|
||||
|
||||
self.addCleanup(reset_context)
|
||||
for unified in (False, True):
|
||||
with self.subTest(unified=unified):
|
||||
reset_context()
|
||||
publish(
|
||||
ServerArgs(model_path="dummy", enable_unified_memory=unified),
|
||||
role="tokenizer",
|
||||
)
|
||||
cc = HybridCacheController.__new__(HybridCacheController)
|
||||
cc.storage_backend = FailingStorage()
|
||||
cc.prefetch_sync_queue = Queue()
|
||||
operation = PrefetchOperation(
|
||||
CacheRequestHandle("r", 0),
|
||||
[1],
|
||||
pool_transfers=[PoolTransfer(name=PoolName.SWA)],
|
||||
)
|
||||
operation.hash_value = ["h0"]
|
||||
operation.prefix_keys = ["prefix"]
|
||||
if unified:
|
||||
with self.assertLogs(level="ERROR"):
|
||||
cc._page_transfer_sidecar(operation, kv_completed_pages=1)
|
||||
ack = cc.prefetch_sync_queue.get_nowait()
|
||||
self.assertIs(ack.operation, operation)
|
||||
self.assertEqual(ack.pool_hits, {})
|
||||
else:
|
||||
with self.assertRaisesRegex(ValueError, "sidecar read failed"):
|
||||
cc._page_transfer_sidecar(operation, kv_completed_pages=1)
|
||||
self.assertTrue(cc.prefetch_sync_queue.empty())
|
||||
self.assertEqual(seen_prefixes, [["prefix", "h0"], ["prefix", "h0"]])
|
||||
reset_context()
|
||||
publish(
|
||||
ServerArgs(model_path="dummy", enable_unified_memory=True),
|
||||
role="tokenizer",
|
||||
)
|
||||
cc = HybridCacheController.__new__(HybridCacheController)
|
||||
cc.storage_backend = FailingStorage()
|
||||
cc.prefetch_sync_queue = Queue()
|
||||
operation = PrefetchOperation(
|
||||
CacheRequestHandle("r", 0),
|
||||
[1],
|
||||
pool_transfers=[PoolTransfer(name=PoolName.SWA)],
|
||||
)
|
||||
operation.hash_value = ["h0"]
|
||||
operation.prefix_keys = ["prefix"]
|
||||
with self.assertLogs(level="ERROR"):
|
||||
cc._page_transfer_sidecar(operation, kv_completed_pages=1)
|
||||
ack = cc.prefetch_sync_queue.get_nowait()
|
||||
self.assertIs(ack.operation, operation)
|
||||
self.assertEqual(ack.pool_hits, {})
|
||||
self.assertTrue(cc.prefetch_sync_queue.empty())
|
||||
self.assertEqual(seen_prefixes, [["prefix", "h0"]])
|
||||
|
||||
def test_shorter_prefetch_reserves_full_and_swa_without_mutating_probe_keys(self):
|
||||
page_size = 4
|
||||
|
||||
Reference in New Issue
Block a user