[HiCache] Keep storage workers alive on backend exceptions (#43550)

This commit is contained in:
Shuwen Wang
2026-10-11 22:26:56 +08:00
committed by GitHub
parent 11bfd868ac
commit ef3e5c83c5
3 changed files with 34 additions and 39 deletions
+10 -6
View File
@@ -43,7 +43,7 @@ from sglang.srt.layers.dp_attention import (
from sglang.srt.mem_cache.l2_transfer import L2Transfer, L2TransferEngine
from sglang.srt.mem_cache.memory_pool import MLATokenToKVPool
from sglang.srt.mem_cache.utils import get_storage_hash_str
from sglang.srt.runtime_context import get_memory, get_parallel
from sglang.srt.runtime_context import get_parallel
from sglang.srt.utils import get_device_module
logger = logging.getLogger(__name__)
@@ -1126,8 +1126,6 @@ class HiCacheController:
kv_derived_transfers,
)
except Exception:
if not get_memory().enable_unified_memory:
raise
logger.exception(
"HiCache prefetch transfer failed for request %s",
operation.request_id,
@@ -1288,8 +1286,6 @@ class HiCacheController:
operation
)
except Exception:
if not get_memory().enable_unified_memory:
raise
logger.exception(
"HiCache storage query failed for request %s",
operation.request_id,
@@ -1374,6 +1370,14 @@ class HiCacheController:
prefix_keys = prefix_keys + batch_hashes
operation.completed_tokens += self.page_size * len(batch_hashes)
def _page_backup_or_log(self, operation):
try:
self._page_backup(operation)
except Exception:
logger.exception(
"HiCache backup failed after %d tokens", operation.completed_tokens
)
def backup_thread_func(self):
"""
Manage backup operations from host memory to storage backend.
@@ -1385,7 +1389,7 @@ class HiCacheController:
continue
if not self.backup_skip:
self._page_backup(operation)
self._page_backup_or_log(operation)
self.ack_backup_queue.put(operation)
except Empty:
@@ -41,7 +41,6 @@ from sglang.srt.mem_cache.pool_host.base import uses_shared_host_layout
from sglang.srt.mem_cache.pool_host.mha import MHATokenToKVPoolHost
from sglang.srt.mem_cache.pool_host.unified import UnifiedPageEnvelopeHostPool
from sglang.srt.mem_cache.radix_cache import RadixKey
from sglang.srt.runtime_context import get_memory
if TYPE_CHECKING:
from sglang.srt.mem_cache.allocator import BaseTokenToKVPoolAllocator
@@ -1513,8 +1512,6 @@ class HybridCacheController(BaseHiCacheController):
)
pool_hits = count_pool_hits(results)
except Exception:
if not get_memory().enable_unified_memory:
raise
logger.exception(
"HiCache sidecar prefetch failed for request %s",
operation.request_id,
@@ -1630,7 +1627,7 @@ class HybridCacheController(BaseHiCacheController):
operation = self.backup_queue.get(block=True, timeout=1)
if operation is None:
continue
self._page_backup(operation)
self._page_backup_or_log(operation)
self.ack_backup_queue.put(operation)
except Empty:
continue