[HiCache] Keep storage workers alive on backend exceptions (#43550)
This commit is contained in:
@@ -43,7 +43,7 @@ from sglang.srt.layers.dp_attention import (
|
||||
from sglang.srt.mem_cache.l2_transfer import L2Transfer, L2TransferEngine
|
||||
from sglang.srt.mem_cache.memory_pool import MLATokenToKVPool
|
||||
from sglang.srt.mem_cache.utils import get_storage_hash_str
|
||||
from sglang.srt.runtime_context import get_memory, get_parallel
|
||||
from sglang.srt.runtime_context import get_parallel
|
||||
from sglang.srt.utils import get_device_module
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -1126,8 +1126,6 @@ class HiCacheController:
|
||||
kv_derived_transfers,
|
||||
)
|
||||
except Exception:
|
||||
if not get_memory().enable_unified_memory:
|
||||
raise
|
||||
logger.exception(
|
||||
"HiCache prefetch transfer failed for request %s",
|
||||
operation.request_id,
|
||||
@@ -1288,8 +1286,6 @@ class HiCacheController:
|
||||
operation
|
||||
)
|
||||
except Exception:
|
||||
if not get_memory().enable_unified_memory:
|
||||
raise
|
||||
logger.exception(
|
||||
"HiCache storage query failed for request %s",
|
||||
operation.request_id,
|
||||
@@ -1374,6 +1370,14 @@ class HiCacheController:
|
||||
prefix_keys = prefix_keys + batch_hashes
|
||||
operation.completed_tokens += self.page_size * len(batch_hashes)
|
||||
|
||||
def _page_backup_or_log(self, operation):
|
||||
try:
|
||||
self._page_backup(operation)
|
||||
except Exception:
|
||||
logger.exception(
|
||||
"HiCache backup failed after %d tokens", operation.completed_tokens
|
||||
)
|
||||
|
||||
def backup_thread_func(self):
|
||||
"""
|
||||
Manage backup operations from host memory to storage backend.
|
||||
@@ -1385,7 +1389,7 @@ class HiCacheController:
|
||||
continue
|
||||
|
||||
if not self.backup_skip:
|
||||
self._page_backup(operation)
|
||||
self._page_backup_or_log(operation)
|
||||
self.ack_backup_queue.put(operation)
|
||||
|
||||
except Empty:
|
||||
|
||||
Reference in New Issue
Block a user