[HiCache] Keep storage workers alive on backend exceptions (#43550)

This commit is contained in:
Shuwen Wang
2026-10-11 22:26:56 +08:00
committed by GitHub
parent 11bfd868ac
commit ef3e5c83c5
3 changed files with 34 additions and 39 deletions
+10 -6
View File
@@ -43,7 +43,7 @@ from sglang.srt.layers.dp_attention import (
from sglang.srt.mem_cache.l2_transfer import L2Transfer, L2TransferEngine
from sglang.srt.mem_cache.memory_pool import MLATokenToKVPool
from sglang.srt.mem_cache.utils import get_storage_hash_str
from sglang.srt.runtime_context import get_memory, get_parallel
from sglang.srt.runtime_context import get_parallel
from sglang.srt.utils import get_device_module
logger = logging.getLogger(__name__)
@@ -1126,8 +1126,6 @@ class HiCacheController:
kv_derived_transfers,
)
except Exception:
if not get_memory().enable_unified_memory:
raise
logger.exception(
"HiCache prefetch transfer failed for request %s",
operation.request_id,
@@ -1288,8 +1286,6 @@ class HiCacheController:
operation
)
except Exception:
if not get_memory().enable_unified_memory:
raise
logger.exception(
"HiCache storage query failed for request %s",
operation.request_id,
@@ -1374,6 +1370,14 @@ class HiCacheController:
prefix_keys = prefix_keys + batch_hashes
operation.completed_tokens += self.page_size * len(batch_hashes)
def _page_backup_or_log(self, operation):
try:
self._page_backup(operation)
except Exception:
logger.exception(
"HiCache backup failed after %d tokens", operation.completed_tokens
)
def backup_thread_func(self):
"""
Manage backup operations from host memory to storage backend.
@@ -1385,7 +1389,7 @@ class HiCacheController:
continue
if not self.backup_skip:
self._page_backup(operation)
self._page_backup_or_log(operation)
self.ack_backup_queue.put(operation)
except Empty:
@@ -41,7 +41,6 @@ from sglang.srt.mem_cache.pool_host.base import uses_shared_host_layout
from sglang.srt.mem_cache.pool_host.mha import MHATokenToKVPoolHost
from sglang.srt.mem_cache.pool_host.unified import UnifiedPageEnvelopeHostPool
from sglang.srt.mem_cache.radix_cache import RadixKey
from sglang.srt.runtime_context import get_memory
if TYPE_CHECKING:
from sglang.srt.mem_cache.allocator import BaseTokenToKVPoolAllocator
@@ -1513,8 +1512,6 @@ class HybridCacheController(BaseHiCacheController):
)
pool_hits = count_pool_hits(results)
except Exception:
if not get_memory().enable_unified_memory:
raise
logger.exception(
"HiCache sidecar prefetch failed for request %s",
operation.request_id,
@@ -1630,7 +1627,7 @@ class HybridCacheController(BaseHiCacheController):
operation = self.backup_queue.get(block=True, timeout=1)
if operation is None:
continue
self._page_backup(operation)
self._page_backup_or_log(operation)
self.ack_backup_queue.put(operation)
except Empty:
continue
@@ -1122,7 +1122,7 @@ def _build_unified_host_pair(bundle):
class TestUnifiedPageEnvelopeHostPool(CustomTestCase):
def test_sidecar_read_error_is_only_a_cache_miss_with_unified_memory(self):
def test_sidecar_read_error_is_a_cache_miss_with_unified_memory(self):
from sglang.srt.runtime_context import publish, reset_context
from sglang.srt.server_args import ServerArgs
@@ -1134,34 +1134,28 @@ class TestUnifiedPageEnvelopeHostPool(CustomTestCase):
raise ValueError("sidecar read failed")
self.addCleanup(reset_context)
for unified in (False, True):
with self.subTest(unified=unified):
reset_context()
publish(
ServerArgs(model_path="dummy", enable_unified_memory=unified),
role="tokenizer",
)
cc = HybridCacheController.__new__(HybridCacheController)
cc.storage_backend = FailingStorage()
cc.prefetch_sync_queue = Queue()
operation = PrefetchOperation(
CacheRequestHandle("r", 0),
[1],
pool_transfers=[PoolTransfer(name=PoolName.SWA)],
)
operation.hash_value = ["h0"]
operation.prefix_keys = ["prefix"]
if unified:
with self.assertLogs(level="ERROR"):
cc._page_transfer_sidecar(operation, kv_completed_pages=1)
ack = cc.prefetch_sync_queue.get_nowait()
self.assertIs(ack.operation, operation)
self.assertEqual(ack.pool_hits, {})
else:
with self.assertRaisesRegex(ValueError, "sidecar read failed"):
cc._page_transfer_sidecar(operation, kv_completed_pages=1)
self.assertTrue(cc.prefetch_sync_queue.empty())
self.assertEqual(seen_prefixes, [["prefix", "h0"], ["prefix", "h0"]])
reset_context()
publish(
ServerArgs(model_path="dummy", enable_unified_memory=True),
role="tokenizer",
)
cc = HybridCacheController.__new__(HybridCacheController)
cc.storage_backend = FailingStorage()
cc.prefetch_sync_queue = Queue()
operation = PrefetchOperation(
CacheRequestHandle("r", 0),
[1],
pool_transfers=[PoolTransfer(name=PoolName.SWA)],
)
operation.hash_value = ["h0"]
operation.prefix_keys = ["prefix"]
with self.assertLogs(level="ERROR"):
cc._page_transfer_sidecar(operation, kv_completed_pages=1)
ack = cc.prefetch_sync_queue.get_nowait()
self.assertIs(ack.operation, operation)
self.assertEqual(ack.pool_hits, {})
self.assertTrue(cc.prefetch_sync_queue.empty())
self.assertEqual(seen_prefixes, [["prefix", "h0"]])
def test_shorter_prefetch_reserves_full_and_swa_without_mutating_probe_keys(self):
page_size = 4