From ef3e5c83c5255690abe2bafb08a128beaed1edc6 Mon Sep 17 00:00:00 2001 From: Shuwen Wang <47200617+alphabetc1@users.noreply.github.com> Date: Sun, 11 Oct 2026 22:26:56 +0800 Subject: [PATCH] [HiCache] Keep storage workers alive on backend exceptions (#43550) --- .../sglang/srt/managers/cache_controller.py | 16 +++--- .../hybrid_cache/hybrid_cache_controller.py | 5 +- .../mem_cache/test_hybrid_pool_assembler.py | 52 ++++++++----------- 3 files changed, 34 insertions(+), 39 deletions(-) diff --git a/python/sglang/srt/managers/cache_controller.py b/python/sglang/srt/managers/cache_controller.py index 8a27dbf38b..4af640c00b 100644 --- a/python/sglang/srt/managers/cache_controller.py +++ b/python/sglang/srt/managers/cache_controller.py @@ -43,7 +43,7 @@ from sglang.srt.layers.dp_attention import ( from sglang.srt.mem_cache.l2_transfer import L2Transfer, L2TransferEngine from sglang.srt.mem_cache.memory_pool import MLATokenToKVPool from sglang.srt.mem_cache.utils import get_storage_hash_str -from sglang.srt.runtime_context import get_memory, get_parallel +from sglang.srt.runtime_context import get_parallel from sglang.srt.utils import get_device_module logger = logging.getLogger(__name__) @@ -1126,8 +1126,6 @@ class HiCacheController: kv_derived_transfers, ) except Exception: - if not get_memory().enable_unified_memory: - raise logger.exception( "HiCache prefetch transfer failed for request %s", operation.request_id, @@ -1288,8 +1286,6 @@ class HiCacheController: operation ) except Exception: - if not get_memory().enable_unified_memory: - raise logger.exception( "HiCache storage query failed for request %s", operation.request_id, @@ -1374,6 +1370,14 @@ class HiCacheController: prefix_keys = prefix_keys + batch_hashes operation.completed_tokens += self.page_size * len(batch_hashes) + def _page_backup_or_log(self, operation): + try: + self._page_backup(operation) + except Exception: + logger.exception( + "HiCache backup failed after %d tokens", operation.completed_tokens + ) + def backup_thread_func(self): """ Manage backup operations from host memory to storage backend. @@ -1385,7 +1389,7 @@ class HiCacheController: continue if not self.backup_skip: - self._page_backup(operation) + self._page_backup_or_log(operation) self.ack_backup_queue.put(operation) except Empty: diff --git a/python/sglang/srt/mem_cache/hybrid_cache/hybrid_cache_controller.py b/python/sglang/srt/mem_cache/hybrid_cache/hybrid_cache_controller.py index a1a2b432c9..7f899abce8 100644 --- a/python/sglang/srt/mem_cache/hybrid_cache/hybrid_cache_controller.py +++ b/python/sglang/srt/mem_cache/hybrid_cache/hybrid_cache_controller.py @@ -41,7 +41,6 @@ from sglang.srt.mem_cache.pool_host.base import uses_shared_host_layout from sglang.srt.mem_cache.pool_host.mha import MHATokenToKVPoolHost from sglang.srt.mem_cache.pool_host.unified import UnifiedPageEnvelopeHostPool from sglang.srt.mem_cache.radix_cache import RadixKey -from sglang.srt.runtime_context import get_memory if TYPE_CHECKING: from sglang.srt.mem_cache.allocator import BaseTokenToKVPoolAllocator @@ -1513,8 +1512,6 @@ class HybridCacheController(BaseHiCacheController): ) pool_hits = count_pool_hits(results) except Exception: - if not get_memory().enable_unified_memory: - raise logger.exception( "HiCache sidecar prefetch failed for request %s", operation.request_id, @@ -1630,7 +1627,7 @@ class HybridCacheController(BaseHiCacheController): operation = self.backup_queue.get(block=True, timeout=1) if operation is None: continue - self._page_backup(operation) + self._page_backup_or_log(operation) self.ack_backup_queue.put(operation) except Empty: continue diff --git a/test/registered/unit/mem_cache/test_hybrid_pool_assembler.py b/test/registered/unit/mem_cache/test_hybrid_pool_assembler.py index 665809ecea..00e8b603a2 100644 --- a/test/registered/unit/mem_cache/test_hybrid_pool_assembler.py +++ b/test/registered/unit/mem_cache/test_hybrid_pool_assembler.py @@ -1122,7 +1122,7 @@ def _build_unified_host_pair(bundle): class TestUnifiedPageEnvelopeHostPool(CustomTestCase): - def test_sidecar_read_error_is_only_a_cache_miss_with_unified_memory(self): + def test_sidecar_read_error_is_a_cache_miss_with_unified_memory(self): from sglang.srt.runtime_context import publish, reset_context from sglang.srt.server_args import ServerArgs @@ -1134,34 +1134,28 @@ class TestUnifiedPageEnvelopeHostPool(CustomTestCase): raise ValueError("sidecar read failed") self.addCleanup(reset_context) - for unified in (False, True): - with self.subTest(unified=unified): - reset_context() - publish( - ServerArgs(model_path="dummy", enable_unified_memory=unified), - role="tokenizer", - ) - cc = HybridCacheController.__new__(HybridCacheController) - cc.storage_backend = FailingStorage() - cc.prefetch_sync_queue = Queue() - operation = PrefetchOperation( - CacheRequestHandle("r", 0), - [1], - pool_transfers=[PoolTransfer(name=PoolName.SWA)], - ) - operation.hash_value = ["h0"] - operation.prefix_keys = ["prefix"] - if unified: - with self.assertLogs(level="ERROR"): - cc._page_transfer_sidecar(operation, kv_completed_pages=1) - ack = cc.prefetch_sync_queue.get_nowait() - self.assertIs(ack.operation, operation) - self.assertEqual(ack.pool_hits, {}) - else: - with self.assertRaisesRegex(ValueError, "sidecar read failed"): - cc._page_transfer_sidecar(operation, kv_completed_pages=1) - self.assertTrue(cc.prefetch_sync_queue.empty()) - self.assertEqual(seen_prefixes, [["prefix", "h0"], ["prefix", "h0"]]) + reset_context() + publish( + ServerArgs(model_path="dummy", enable_unified_memory=True), + role="tokenizer", + ) + cc = HybridCacheController.__new__(HybridCacheController) + cc.storage_backend = FailingStorage() + cc.prefetch_sync_queue = Queue() + operation = PrefetchOperation( + CacheRequestHandle("r", 0), + [1], + pool_transfers=[PoolTransfer(name=PoolName.SWA)], + ) + operation.hash_value = ["h0"] + operation.prefix_keys = ["prefix"] + with self.assertLogs(level="ERROR"): + cc._page_transfer_sidecar(operation, kv_completed_pages=1) + ack = cc.prefetch_sync_queue.get_nowait() + self.assertIs(ack.operation, operation) + self.assertEqual(ack.pool_hits, {}) + self.assertTrue(cc.prefetch_sync_queue.empty()) + self.assertEqual(seen_prefixes, [["prefix", "h0"]]) def test_shorter_prefetch_reserves_full_and_swa_without_mutating_probe_keys(self): page_size = 4